drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28
  29 #include <mlx5_glue.h>
  30 #include <mlx5_prm.h>
  31
  32 #include "mlx5_defs.h"
  33 #include "mlx5.h"
  34 #include "mlx5_flow.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #define VERBS_SPEC_INNER(item_flags) \
  38         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  39
  40 /**
  41  * Get Verbs flow counter by index.
  42  *
  43  * @param[in] dev
  44  *   Pointer to the Ethernet device structure.
  45  * @param[in] idx
  46  *   mlx5 flow counter index in the container.
  47  * @param[out] ppool
  48  *   mlx5 flow counter pool in the container,
  49  *
  50  * @return
  51  *   A pointer to the counter, NULL otherwise.
  52  */
  53 static struct mlx5_flow_counter *
  54 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
  55                               uint32_t idx,
  56                               struct mlx5_flow_counter_pool **ppool)
  57 {
  58         struct mlx5_priv *priv = dev->data->dev_private;
  59         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0,
  60                                                                         0);
  61         struct mlx5_flow_counter_pool *pool;
  62
  63         idx--;
  64         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
  65         MLX5_ASSERT(pool);
  66         if (ppool)
  67                 *ppool = pool;
  68         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
  69 }
  70
  71 /**
  72  * Create Verbs flow counter with Verbs library.
  73  *
  74  * @param[in] dev
  75  *   Pointer to the Ethernet device structure.
  76  * @param[in, out] counter
  77  *   mlx5 flow counter object, contains the counter id,
  78  *   handle of created Verbs flow counter is returned
  79  *   in cs field (if counters are supported).
  80  *
  81  * @return
  82  *   0 On success else a negative errno value is returned
  83  *   and rte_errno is set.
  84  */
  85 static int
  86 flow_verbs_counter_create(struct rte_eth_dev *dev,
  87                           struct mlx5_flow_counter_ext *counter)
  88 {
  89 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  90         struct mlx5_priv *priv = dev->data->dev_private;
  91         struct ibv_context *ctx = priv->sh->ctx;
  92         struct ibv_counter_set_init_attr init = {
  93                          .counter_set_id = counter->id};
  94
  95         counter->cs = mlx5_glue->create_counter_set(ctx, &init);
  96         if (!counter->cs) {
  97                 rte_errno = ENOTSUP;
  98                 return -ENOTSUP;
  99         }
 100         return 0;
 101 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 102         struct mlx5_priv *priv = dev->data->dev_private;
 103         struct ibv_context *ctx = priv->sh->ctx;
 104         struct ibv_counters_init_attr init = {0};
 105         struct ibv_counter_attach_attr attach;
 106         int ret;
 107
 108         memset(&attach, 0, sizeof(attach));
 109         counter->cs = mlx5_glue->create_counters(ctx, &init);
 110         if (!counter->cs) {
 111                 rte_errno = ENOTSUP;
 112                 return -ENOTSUP;
 113         }
 114         attach.counter_desc = IBV_COUNTER_PACKETS;
 115         attach.index = 0;
 116         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
 117         if (!ret) {
 118                 attach.counter_desc = IBV_COUNTER_BYTES;
 119                 attach.index = 1;
 120                 ret = mlx5_glue->attach_counters
 121                                         (counter->cs, &attach, NULL);
 122         }
 123         if (ret) {
 124                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 125                 counter->cs = NULL;
 126                 rte_errno = ret;
 127                 return -ret;
 128         }
 129         return 0;
 130 #else
 131         (void)dev;
 132         (void)counter;
 133         rte_errno = ENOTSUP;
 134         return -ENOTSUP;
 135 #endif
 136 }
 137
 138 /**
 139  * Get a flow counter.
 140  *
 141  * @param[in] dev
 142  *   Pointer to the Ethernet device structure.
 143  * @param[in] shared
 144  *   Indicate if this counter is shared with other flows.
 145  * @param[in] id
 146  *   Counter identifier.
 147  *
 148  * @return
 149  *   Index to the counter, 0 otherwise and rte_errno is set.
 150  */
 151 static uint32_t
 152 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 153 {
 154         struct mlx5_priv *priv = dev->data->dev_private;
 155         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0,
 156                                                                         0);
 157         struct mlx5_flow_counter_pool *pool = NULL;
 158         struct mlx5_flow_counter_ext *cnt_ext = NULL;
 159         struct mlx5_flow_counter *cnt = NULL;
 160         uint32_t n_valid = rte_atomic16_read(&cont->n_valid);
 161         uint32_t pool_idx;
 162         uint32_t i;
 163         int ret;
 164
 165         if (shared) {
 166                 for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 167                         pool = cont->pools[pool_idx];
 168                         for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
 169                                 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
 170                                 if (cnt_ext->shared && cnt_ext->id == id) {
 171                                         cnt_ext->ref_cnt++;
 172                                         return MLX5_MAKE_CNT_IDX(pool_idx, i);
 173                                 }
 174                         }
 175                 }
 176         }
 177         for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 178                 pool = cont->pools[pool_idx];
 179                 if (!pool)
 180                         continue;
 181                 cnt = TAILQ_FIRST(&pool->counters);
 182                 if (cnt)
 183                         break;
 184         }
 185         if (!cnt) {
 186                 struct mlx5_flow_counter_pool **pools;
 187                 uint32_t size;
 188
 189                 if (n_valid == cont->n) {
 190                         /* Resize the container pool array. */
 191                         size = sizeof(struct mlx5_flow_counter_pool *) *
 192                                      (n_valid + MLX5_CNT_CONTAINER_RESIZE);
 193                         pools = rte_zmalloc(__func__, size, 0);
 194                         if (!pools)
 195                                 return 0;
 196                         if (n_valid) {
 197                                 memcpy(pools, cont->pools,
 198                                        sizeof(struct mlx5_flow_counter_pool *) *
 199                                        n_valid);
 200                                 rte_free(cont->pools);
 201                         }
 202                         cont->pools = pools;
 203                         cont->n += MLX5_CNT_CONTAINER_RESIZE;
 204                 }
 205                 /* Allocate memory for new pool*/
 206                 size = sizeof(*pool) + sizeof(*cnt_ext) *
 207                        MLX5_COUNTERS_PER_POOL;
 208                 pool = rte_calloc(__func__, 1, size, 0);
 209                 if (!pool)
 210                         return 0;
 211                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
 212                         cnt = MLX5_POOL_GET_CNT(pool, i);
 213                         TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
 214                 }
 215                 cnt = MLX5_POOL_GET_CNT(pool, 0);
 216                 cont->pools[n_valid] = pool;
 217                 pool_idx = n_valid;
 218                 rte_atomic16_add(&cont->n_valid, 1);
 219                 TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
 220         }
 221         i = MLX5_CNT_ARRAY_IDX(pool, cnt);
 222         cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
 223         cnt_ext->id = id;
 224         cnt_ext->shared = shared;
 225         cnt_ext->ref_cnt = 1;
 226         cnt->hits = 0;
 227         cnt->bytes = 0;
 228         /* Create counter with Verbs. */
 229         ret = flow_verbs_counter_create(dev, cnt_ext);
 230         if (!ret) {
 231                 TAILQ_REMOVE(&pool->counters, cnt, next);
 232                 return MLX5_MAKE_CNT_IDX(pool_idx, i);
 233         }
 234         /* Some error occurred in Verbs library. */
 235         rte_errno = -ret;
 236         return 0;
 237 }
 238
 239 /**
 240  * Release a flow counter.
 241  *
 242  * @param[in] dev
 243  *   Pointer to the Ethernet device structure.
 244  * @param[in] counter
 245  *   Index to the counter handler.
 246  */
 247 static void
 248 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
 249 {
 250         struct mlx5_flow_counter_pool *pool;
 251         struct mlx5_flow_counter *cnt;
 252         struct mlx5_flow_counter_ext *cnt_ext;
 253
 254         cnt = flow_verbs_counter_get_by_idx(dev, counter,
 255                                             &pool);
 256         cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
 257         if (--cnt_ext->ref_cnt == 0) {
 258 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 259                 claim_zero(mlx5_glue->destroy_counter_set(cnt_ext->cs));
 260                 cnt_ext->cs = NULL;
 261 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 262                 claim_zero(mlx5_glue->destroy_counters(cnt_ext->cs));
 263                 cnt_ext->cs = NULL;
 264 #endif
 265                 TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
 266         }
 267 }
 268
 269 /**
 270  * Query a flow counter via Verbs library call.
 271  *
 272  * @see rte_flow_query()
 273  * @see rte_flow_ops
 274  */
 275 static int
 276 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 277                          struct rte_flow *flow, void *data,
 278                          struct rte_flow_error *error)
 279 {
 280 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 281         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 282         if (flow->counter) {
 283                 struct mlx5_flow_counter_pool *pool;
 284                 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
 285                                                 (dev, flow->counter, &pool);
 286                 struct mlx5_flow_counter_ext *cnt_ext = MLX5_CNT_TO_CNT_EXT
 287                                                 (pool, cnt);
 288                 struct rte_flow_query_count *qc = data;
 289                 uint64_t counters[2] = {0, 0};
 290 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 291                 struct ibv_query_counter_set_attr query_cs_attr = {
 292                         .cs = cnt_ext->cs,
 293                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 294                 };
 295                 struct ibv_counter_set_data query_out = {
 296                         .out = counters,
 297                         .outlen = 2 * sizeof(uint64_t),
 298                 };
 299                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 300                                                        &query_out);
 301 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 302                 int err = mlx5_glue->query_counters
 303                                (cnt_ext->cs, counters,
 304                                 RTE_DIM(counters),
 305                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 306 #endif
 307                 if (err)
 308                         return rte_flow_error_set
 309                                 (error, err,
 310                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 311                                  NULL,
 312                                  "cannot read counter");
 313                 qc->hits_set = 1;
 314                 qc->bytes_set = 1;
 315                 qc->hits = counters[0] - cnt->hits;
 316                 qc->bytes = counters[1] - cnt->bytes;
 317                 if (qc->reset) {
 318                         cnt->hits = counters[0];
 319                         cnt->bytes = counters[1];
 320                 }
 321                 return 0;
 322         }
 323         return rte_flow_error_set(error, EINVAL,
 324                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 325                                   NULL,
 326                                   "flow does not have counter");
 327 #else
 328         (void)flow;
 329         (void)data;
 330         return rte_flow_error_set(error, ENOTSUP,
 331                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 332                                   NULL,
 333                                   "counters are not available");
 334 #endif
 335 }
 336
 337 /**
 338  * Add a verbs item specification into @p verbs.
 339  *
 340  * @param[out] verbs
 341  *   Pointer to verbs structure.
 342  * @param[in] src
 343  *   Create specification.
 344  * @param[in] size
 345  *   Size in bytes of the specification to copy.
 346  */
 347 static void
 348 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
 349                     void *src, unsigned int size)
 350 {
 351         void *dst;
 352
 353         if (!verbs)
 354                 return;
 355         MLX5_ASSERT(verbs->specs);
 356         dst = (void *)(verbs->specs + verbs->size);
 357         memcpy(dst, src, size);
 358         ++verbs->attr.num_of_specs;
 359         verbs->size += size;
 360 }
 361
 362 /**
 363  * Convert the @p item into a Verbs specification. This function assumes that
 364  * the input is valid and that there is space to insert the requested item
 365  * into the flow.
 366  *
 367  * @param[in, out] dev_flow
 368  *   Pointer to dev_flow structure.
 369  * @param[in] item
 370  *   Item specification.
 371  * @param[in] item_flags
 372  *   Parsed item flags.
 373  */
 374 static void
 375 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 376                               const struct rte_flow_item *item,
 377                               uint64_t item_flags)
 378 {
 379         const struct rte_flow_item_eth *spec = item->spec;
 380         const struct rte_flow_item_eth *mask = item->mask;
 381         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 382         struct ibv_flow_spec_eth eth = {
 383                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 384                 .size = size,
 385         };
 386
 387         if (!mask)
 388                 mask = &rte_flow_item_eth_mask;
 389         if (spec) {
 390                 unsigned int i;
 391
 392                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 393                         RTE_ETHER_ADDR_LEN);
 394                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 395                         RTE_ETHER_ADDR_LEN);
 396                 eth.val.ether_type = spec->type;
 397                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 398                         RTE_ETHER_ADDR_LEN);
 399                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 400                         RTE_ETHER_ADDR_LEN);
 401                 eth.mask.ether_type = mask->type;
 402                 /* Remove unwanted bits from values. */
 403                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 404                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 405                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 406                 }
 407                 eth.val.ether_type &= eth.mask.ether_type;
 408         }
 409         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 410 }
 411
 412 /**
 413  * Update the VLAN tag in the Verbs Ethernet specification.
 414  * This function assumes that the input is valid and there is space to add
 415  * the requested item.
 416  *
 417  * @param[in, out] attr
 418  *   Pointer to Verbs attributes structure.
 419  * @param[in] eth
 420  *   Verbs structure containing the VLAN information to copy.
 421  */
 422 static void
 423 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 424                             struct ibv_flow_spec_eth *eth)
 425 {
 426         unsigned int i;
 427         const enum ibv_flow_spec_type search = eth->type;
 428         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 429                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 430
 431         for (i = 0; i != attr->num_of_specs; ++i) {
 432                 if (hdr->type == search) {
 433                         struct ibv_flow_spec_eth *e =
 434                                 (struct ibv_flow_spec_eth *)hdr;
 435
 436                         e->val.vlan_tag = eth->val.vlan_tag;
 437                         e->mask.vlan_tag = eth->mask.vlan_tag;
 438                         e->val.ether_type = eth->val.ether_type;
 439                         e->mask.ether_type = eth->mask.ether_type;
 440                         break;
 441                 }
 442                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 443         }
 444 }
 445
 446 /**
 447  * Convert the @p item into a Verbs specification. This function assumes that
 448  * the input is valid and that there is space to insert the requested item
 449  * into the flow.
 450  *
 451  * @param[in, out] dev_flow
 452  *   Pointer to dev_flow structure.
 453  * @param[in] item
 454  *   Item specification.
 455  * @param[in] item_flags
 456  *   Parsed item flags.
 457  */
 458 static void
 459 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 460                                const struct rte_flow_item *item,
 461                                uint64_t item_flags)
 462 {
 463         const struct rte_flow_item_vlan *spec = item->spec;
 464         const struct rte_flow_item_vlan *mask = item->mask;
 465         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 466         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 467         struct ibv_flow_spec_eth eth = {
 468                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 469                 .size = size,
 470         };
 471         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 472                                       MLX5_FLOW_LAYER_OUTER_L2;
 473
 474         if (!mask)
 475                 mask = &rte_flow_item_vlan_mask;
 476         if (spec) {
 477                 eth.val.vlan_tag = spec->tci;
 478                 eth.mask.vlan_tag = mask->tci;
 479                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 480                 eth.val.ether_type = spec->inner_type;
 481                 eth.mask.ether_type = mask->inner_type;
 482                 eth.val.ether_type &= eth.mask.ether_type;
 483         }
 484         if (!(item_flags & l2m))
 485                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 486         else
 487                 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, &eth);
 488         if (!tunnel)
 489                 dev_flow->handle->vf_vlan.tag =
 490                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 491 }
 492
 493 /**
 494  * Convert the @p item into a Verbs specification. This function assumes that
 495  * the input is valid and that there is space to insert the requested item
 496  * into the flow.
 497  *
 498  * @param[in, out] dev_flow
 499  *   Pointer to dev_flow structure.
 500  * @param[in] item
 501  *   Item specification.
 502  * @param[in] item_flags
 503  *   Parsed item flags.
 504  */
 505 static void
 506 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 507                                const struct rte_flow_item *item,
 508                                uint64_t item_flags)
 509 {
 510         const struct rte_flow_item_ipv4 *spec = item->spec;
 511         const struct rte_flow_item_ipv4 *mask = item->mask;
 512         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 513         struct ibv_flow_spec_ipv4_ext ipv4 = {
 514                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 515                 .size = size,
 516         };
 517
 518         if (!mask)
 519                 mask = &rte_flow_item_ipv4_mask;
 520         if (spec) {
 521                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 522                         .src_ip = spec->hdr.src_addr,
 523                         .dst_ip = spec->hdr.dst_addr,
 524                         .proto = spec->hdr.next_proto_id,
 525                         .tos = spec->hdr.type_of_service,
 526                 };
 527                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 528                         .src_ip = mask->hdr.src_addr,
 529                         .dst_ip = mask->hdr.dst_addr,
 530                         .proto = mask->hdr.next_proto_id,
 531                         .tos = mask->hdr.type_of_service,
 532                 };
 533                 /* Remove unwanted bits from values. */
 534                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 535                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 536                 ipv4.val.proto &= ipv4.mask.proto;
 537                 ipv4.val.tos &= ipv4.mask.tos;
 538         }
 539         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 540 }
 541
 542 /**
 543  * Convert the @p item into a Verbs specification. This function assumes that
 544  * the input is valid and that there is space to insert the requested item
 545  * into the flow.
 546  *
 547  * @param[in, out] dev_flow
 548  *   Pointer to dev_flow structure.
 549  * @param[in] item
 550  *   Item specification.
 551  * @param[in] item_flags
 552  *   Parsed item flags.
 553  */
 554 static void
 555 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 556                                const struct rte_flow_item *item,
 557                                uint64_t item_flags)
 558 {
 559         const struct rte_flow_item_ipv6 *spec = item->spec;
 560         const struct rte_flow_item_ipv6 *mask = item->mask;
 561         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 562         struct ibv_flow_spec_ipv6 ipv6 = {
 563                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 564                 .size = size,
 565         };
 566
 567         if (!mask)
 568                 mask = &rte_flow_item_ipv6_mask;
 569         if (spec) {
 570                 unsigned int i;
 571                 uint32_t vtc_flow_val;
 572                 uint32_t vtc_flow_mask;
 573
 574                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 575                        RTE_DIM(ipv6.val.src_ip));
 576                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 577                        RTE_DIM(ipv6.val.dst_ip));
 578                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 579                        RTE_DIM(ipv6.mask.src_ip));
 580                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 581                        RTE_DIM(ipv6.mask.dst_ip));
 582                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 583                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 584                 ipv6.val.flow_label =
 585                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 586                                          RTE_IPV6_HDR_FL_SHIFT);
 587                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 588                                          RTE_IPV6_HDR_TC_SHIFT;
 589                 ipv6.val.next_hdr = spec->hdr.proto;
 590                 ipv6.mask.flow_label =
 591                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 592                                          RTE_IPV6_HDR_FL_SHIFT);
 593                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 594                                           RTE_IPV6_HDR_TC_SHIFT;
 595                 ipv6.mask.next_hdr = mask->hdr.proto;
 596                 /* Remove unwanted bits from values. */
 597                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 598                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 599                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 600                 }
 601                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 602                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 603                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 604         }
 605         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 606 }
 607
 608 /**
 609  * Convert the @p item into a Verbs specification. This function assumes that
 610  * the input is valid and that there is space to insert the requested item
 611  * into the flow.
 612  *
 613  * @param[in, out] dev_flow
 614  *   Pointer to dev_flow structure.
 615  * @param[in] item
 616  *   Item specification.
 617  * @param[in] item_flags
 618  *   Parsed item flags.
 619  */
 620 static void
 621 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 622                               const struct rte_flow_item *item,
 623                               uint64_t item_flags __rte_unused)
 624 {
 625         const struct rte_flow_item_tcp *spec = item->spec;
 626         const struct rte_flow_item_tcp *mask = item->mask;
 627         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 628         struct ibv_flow_spec_tcp_udp tcp = {
 629                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 630                 .size = size,
 631         };
 632
 633         if (!mask)
 634                 mask = &rte_flow_item_tcp_mask;
 635         if (spec) {
 636                 tcp.val.dst_port = spec->hdr.dst_port;
 637                 tcp.val.src_port = spec->hdr.src_port;
 638                 tcp.mask.dst_port = mask->hdr.dst_port;
 639                 tcp.mask.src_port = mask->hdr.src_port;
 640                 /* Remove unwanted bits from values. */
 641                 tcp.val.src_port &= tcp.mask.src_port;
 642                 tcp.val.dst_port &= tcp.mask.dst_port;
 643         }
 644         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 645 }
 646
 647 /**
 648  * Convert the @p item into a Verbs specification. This function assumes that
 649  * the input is valid and that there is space to insert the requested item
 650  * into the flow.
 651  *
 652  * @param[in, out] dev_flow
 653  *   Pointer to dev_flow structure.
 654  * @param[in] item
 655  *   Item specification.
 656  * @param[in] item_flags
 657  *   Parsed item flags.
 658  */
 659 static void
 660 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 661                               const struct rte_flow_item *item,
 662                               uint64_t item_flags __rte_unused)
 663 {
 664         const struct rte_flow_item_udp *spec = item->spec;
 665         const struct rte_flow_item_udp *mask = item->mask;
 666         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 667         struct ibv_flow_spec_tcp_udp udp = {
 668                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 669                 .size = size,
 670         };
 671
 672         if (!mask)
 673                 mask = &rte_flow_item_udp_mask;
 674         if (spec) {
 675                 udp.val.dst_port = spec->hdr.dst_port;
 676                 udp.val.src_port = spec->hdr.src_port;
 677                 udp.mask.dst_port = mask->hdr.dst_port;
 678                 udp.mask.src_port = mask->hdr.src_port;
 679                 /* Remove unwanted bits from values. */
 680                 udp.val.src_port &= udp.mask.src_port;
 681                 udp.val.dst_port &= udp.mask.dst_port;
 682         }
 683         item++;
 684         while (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 685                 item++;
 686         if (!(udp.val.dst_port & udp.mask.dst_port)) {
 687                 switch ((item)->type) {
 688                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 689                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN);
 690                         udp.mask.dst_port = 0xffff;
 691                         break;
 692                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 693                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE);
 694                         udp.mask.dst_port = 0xffff;
 695                         break;
 696                 case RTE_FLOW_ITEM_TYPE_MPLS:
 697                         udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS);
 698                         udp.mask.dst_port = 0xffff;
 699                         break;
 700                 default:
 701                         break;
 702                 }
 703         }
 704
 705         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 706 }
 707
 708 /**
 709  * Convert the @p item into a Verbs specification. This function assumes that
 710  * the input is valid and that there is space to insert the requested item
 711  * into the flow.
 712  *
 713  * @param[in, out] dev_flow
 714  *   Pointer to dev_flow structure.
 715  * @param[in] item
 716  *   Item specification.
 717  * @param[in] item_flags
 718  *   Parsed item flags.
 719  */
 720 static void
 721 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 722                                 const struct rte_flow_item *item,
 723                                 uint64_t item_flags __rte_unused)
 724 {
 725         const struct rte_flow_item_vxlan *spec = item->spec;
 726         const struct rte_flow_item_vxlan *mask = item->mask;
 727         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 728         struct ibv_flow_spec_tunnel vxlan = {
 729                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 730                 .size = size,
 731         };
 732         union vni {
 733                 uint32_t vlan_id;
 734                 uint8_t vni[4];
 735         } id = { .vlan_id = 0, };
 736
 737         if (!mask)
 738                 mask = &rte_flow_item_vxlan_mask;
 739         if (spec) {
 740                 memcpy(&id.vni[1], spec->vni, 3);
 741                 vxlan.val.tunnel_id = id.vlan_id;
 742                 memcpy(&id.vni[1], mask->vni, 3);
 743                 vxlan.mask.tunnel_id = id.vlan_id;
 744                 /* Remove unwanted bits from values. */
 745                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 746         }
 747         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 748 }
 749
 750 /**
 751  * Convert the @p item into a Verbs specification. This function assumes that
 752  * the input is valid and that there is space to insert the requested item
 753  * into the flow.
 754  *
 755  * @param[in, out] dev_flow
 756  *   Pointer to dev_flow structure.
 757  * @param[in] item
 758  *   Item specification.
 759  * @param[in] item_flags
 760  *   Parsed item flags.
 761  */
 762 static void
 763 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 764                                     const struct rte_flow_item *item,
 765                                     uint64_t item_flags __rte_unused)
 766 {
 767         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 768         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 769         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 770         struct ibv_flow_spec_tunnel vxlan_gpe = {
 771                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 772                 .size = size,
 773         };
 774         union vni {
 775                 uint32_t vlan_id;
 776                 uint8_t vni[4];
 777         } id = { .vlan_id = 0, };
 778
 779         if (!mask)
 780                 mask = &rte_flow_item_vxlan_gpe_mask;
 781         if (spec) {
 782                 memcpy(&id.vni[1], spec->vni, 3);
 783                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 784                 memcpy(&id.vni[1], mask->vni, 3);
 785                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 786                 /* Remove unwanted bits from values. */
 787                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 788         }
 789         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 790 }
 791
 792 /**
 793  * Update the protocol in Verbs IPv4/IPv6 spec.
 794  *
 795  * @param[in, out] attr
 796  *   Pointer to Verbs attributes structure.
 797  * @param[in] search
 798  *   Specification type to search in order to update the IP protocol.
 799  * @param[in] protocol
 800  *   Protocol value to set if none is present in the specification.
 801  */
 802 static void
 803 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 804                                        enum ibv_flow_spec_type search,
 805                                        uint8_t protocol)
 806 {
 807         unsigned int i;
 808         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 809                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 810
 811         if (!attr)
 812                 return;
 813         for (i = 0; i != attr->num_of_specs; ++i) {
 814                 if (hdr->type == search) {
 815                         union {
 816                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 817                                 struct ibv_flow_spec_ipv6 *ipv6;
 818                         } ip;
 819
 820                         switch (search) {
 821                         case IBV_FLOW_SPEC_IPV4_EXT:
 822                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 823                                 if (!ip.ipv4->val.proto) {
 824                                         ip.ipv4->val.proto = protocol;
 825                                         ip.ipv4->mask.proto = 0xff;
 826                                 }
 827                                 break;
 828                         case IBV_FLOW_SPEC_IPV6:
 829                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 830                                 if (!ip.ipv6->val.next_hdr) {
 831                                         ip.ipv6->val.next_hdr = protocol;
 832                                         ip.ipv6->mask.next_hdr = 0xff;
 833                                 }
 834                                 break;
 835                         default:
 836                                 break;
 837                         }
 838                         break;
 839                 }
 840                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 841         }
 842 }
 843
 844 /**
 845  * Convert the @p item into a Verbs specification. This function assumes that
 846  * the input is valid and that there is space to insert the requested item
 847  * into the flow.
 848  *
 849  * @param[in, out] dev_flow
 850  *   Pointer to dev_flow structure.
 851  * @param[in] item
 852  *   Item specification.
 853  * @param[in] item_flags
 854  *   Parsed item flags.
 855  */
 856 static void
 857 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 858                               const struct rte_flow_item *item __rte_unused,
 859                               uint64_t item_flags)
 860 {
 861         struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
 862 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 863         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 864         struct ibv_flow_spec_tunnel tunnel = {
 865                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 866                 .size = size,
 867         };
 868 #else
 869         const struct rte_flow_item_gre *spec = item->spec;
 870         const struct rte_flow_item_gre *mask = item->mask;
 871         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 872         struct ibv_flow_spec_gre tunnel = {
 873                 .type = IBV_FLOW_SPEC_GRE,
 874                 .size = size,
 875         };
 876
 877         if (!mask)
 878                 mask = &rte_flow_item_gre_mask;
 879         if (spec) {
 880                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 881                 tunnel.val.protocol = spec->protocol;
 882                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 883                 tunnel.mask.protocol = mask->protocol;
 884                 /* Remove unwanted bits from values. */
 885                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 886                 tunnel.val.protocol &= tunnel.mask.protocol;
 887                 tunnel.val.key &= tunnel.mask.key;
 888         }
 889 #endif
 890         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 891                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 892                                                        IBV_FLOW_SPEC_IPV4_EXT,
 893                                                        IPPROTO_GRE);
 894         else
 895                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 896                                                        IBV_FLOW_SPEC_IPV6,
 897                                                        IPPROTO_GRE);
 898         flow_verbs_spec_add(verbs, &tunnel, size);
 899 }
 900
 901 /**
 902  * Convert the @p action into a Verbs specification. This function assumes that
 903  * the input is valid and that there is space to insert the requested action
 904  * into the flow. This function also return the action that was added.
 905  *
 906  * @param[in, out] dev_flow
 907  *   Pointer to dev_flow structure.
 908  * @param[in] item
 909  *   Item specification.
 910  * @param[in] item_flags
 911  *   Parsed item flags.
 912  */
 913 static void
 914 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
 915                                const struct rte_flow_item *item __rte_unused,
 916                                uint64_t item_flags __rte_unused)
 917 {
 918 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 919         const struct rte_flow_item_mpls *spec = item->spec;
 920         const struct rte_flow_item_mpls *mask = item->mask;
 921         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 922         struct ibv_flow_spec_mpls mpls = {
 923                 .type = IBV_FLOW_SPEC_MPLS,
 924                 .size = size,
 925         };
 926
 927         if (!mask)
 928                 mask = &rte_flow_item_mpls_mask;
 929         if (spec) {
 930                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 931                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 932                 /* Remove unwanted bits from values.  */
 933                 mpls.val.label &= mpls.mask.label;
 934         }
 935         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
 936 #endif
 937 }
 938
 939 /**
 940  * Convert the @p action into a Verbs specification. This function assumes that
 941  * the input is valid and that there is space to insert the requested action
 942  * into the flow.
 943  *
 944  * @param[in] dev_flow
 945  *   Pointer to mlx5_flow.
 946  * @param[in] action
 947  *   Action configuration.
 948  */
 949 static void
 950 flow_verbs_translate_action_drop
 951         (struct mlx5_flow *dev_flow,
 952          const struct rte_flow_action *action __rte_unused)
 953 {
 954         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 955         struct ibv_flow_spec_action_drop drop = {
 956                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 957                         .size = size,
 958         };
 959
 960         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
 961 }
 962
 963 /**
 964  * Convert the @p action into a Verbs specification. This function assumes that
 965  * the input is valid and that there is space to insert the requested action
 966  * into the flow.
 967  *
 968  * @param[in] rss_desc
 969  *   Pointer to mlx5_flow_rss_desc.
 970  * @param[in] action
 971  *   Action configuration.
 972  */
 973 static void
 974 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc,
 975                                   const struct rte_flow_action *action)
 976 {
 977         const struct rte_flow_action_queue *queue = action->conf;
 978
 979         rss_desc->queue[0] = queue->index;
 980         rss_desc->queue_num = 1;
 981 }
 982
 983 /**
 984  * Convert the @p action into a Verbs specification. This function assumes that
 985  * the input is valid and that there is space to insert the requested action
 986  * into the flow.
 987  *
 988  * @param[in] rss_desc
 989  *   Pointer to mlx5_flow_rss_desc.
 990  * @param[in] action
 991  *   Action configuration.
 992  */
 993 static void
 994 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc,
 995                                 const struct rte_flow_action *action)
 996 {
 997         const struct rte_flow_action_rss *rss = action->conf;
 998         const uint8_t *rss_key;
 999
1000         memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t));
1001         rss_desc->queue_num = rss->queue_num;
1002         /* NULL RSS key indicates default RSS key. */
1003         rss_key = !rss->key ? rss_hash_default_key : rss->key;
1004         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
1005         /*
1006          * rss->level and rss.types should be set in advance when expanding
1007          * items for RSS.
1008          */
1009 }
1010
1011 /**
1012  * Convert the @p action into a Verbs specification. This function assumes that
1013  * the input is valid and that there is space to insert the requested action
1014  * into the flow.
1015  *
1016  * @param[in] dev_flow
1017  *   Pointer to mlx5_flow.
1018  * @param[in] action
1019  *   Action configuration.
1020  */
1021 static void
1022 flow_verbs_translate_action_flag
1023         (struct mlx5_flow *dev_flow,
1024          const struct rte_flow_action *action __rte_unused)
1025 {
1026         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1027         struct ibv_flow_spec_action_tag tag = {
1028                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1029                 .size = size,
1030                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1031         };
1032
1033         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1034 }
1035
1036 /**
1037  * Convert the @p action into a Verbs specification. This function assumes that
1038  * the input is valid and that there is space to insert the requested action
1039  * into the flow.
1040  *
1041  * @param[in] dev_flow
1042  *   Pointer to mlx5_flow.
1043  * @param[in] action
1044  *   Action configuration.
1045  */
1046 static void
1047 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1048                                  const struct rte_flow_action *action)
1049 {
1050         const struct rte_flow_action_mark *mark = action->conf;
1051         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1052         struct ibv_flow_spec_action_tag tag = {
1053                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1054                 .size = size,
1055                 .tag_id = mlx5_flow_mark_set(mark->id),
1056         };
1057
1058         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1059 }
1060
1061 /**
1062  * Convert the @p action into a Verbs specification. This function assumes that
1063  * the input is valid and that there is space to insert the requested action
1064  * into the flow.
1065  *
1066  * @param[in] dev
1067  *   Pointer to the Ethernet device structure.
1068  * @param[in] action
1069  *   Action configuration.
1070  * @param[in] dev_flow
1071  *   Pointer to mlx5_flow.
1072  * @param[out] error
1073  *   Pointer to error structure.
1074  *
1075  * @return
1076  *   0 On success else a negative errno value is returned and rte_errno is set.
1077  */
1078 static int
1079 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1080                                   const struct rte_flow_action *action,
1081                                   struct rte_eth_dev *dev,
1082                                   struct rte_flow_error *error)
1083 {
1084         const struct rte_flow_action_count *count = action->conf;
1085         struct rte_flow *flow = dev_flow->flow;
1086 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1087         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1088         struct mlx5_flow_counter_pool *pool;
1089         struct mlx5_flow_counter *cnt = NULL;
1090         struct mlx5_flow_counter_ext *cnt_ext;
1091         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1092         struct ibv_flow_spec_counter_action counter = {
1093                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1094                 .size = size,
1095         };
1096 #endif
1097
1098         if (!flow->counter) {
1099                 flow->counter = flow_verbs_counter_new(dev, count->shared,
1100                                                        count->id);
1101                 if (!flow->counter)
1102                         return rte_flow_error_set(error, rte_errno,
1103                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1104                                                   action,
1105                                                   "cannot get counter"
1106                                                   " context.");
1107         }
1108 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1109         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1110         cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1111         counter.counter_set_handle = cnt_ext->cs->handle;
1112         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1113 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1114         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1115         cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
1116         counter.counters = cnt_ext->cs;
1117         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1118 #endif
1119         return 0;
1120 }
1121
1122 /**
1123  * Internal validation function. For validating both actions and items.
1124  *
1125  * @param[in] dev
1126  *   Pointer to the Ethernet device structure.
1127  * @param[in] attr
1128  *   Pointer to the flow attributes.
1129  * @param[in] items
1130  *   Pointer to the list of items.
1131  * @param[in] actions
1132  *   Pointer to the list of actions.
1133  * @param[in] external
1134  *   This flow rule is created by request external to PMD.
1135  * @param[in] hairpin
1136  *   Number of hairpin TX actions, 0 means classic flow.
1137  * @param[out] error
1138  *   Pointer to the error structure.
1139  *
1140  * @return
1141  *   0 on success, a negative errno value otherwise and rte_errno is set.
1142  */
1143 static int
1144 flow_verbs_validate(struct rte_eth_dev *dev,
1145                     const struct rte_flow_attr *attr,
1146                     const struct rte_flow_item items[],
1147                     const struct rte_flow_action actions[],
1148                     bool external __rte_unused,
1149                     int hairpin __rte_unused,
1150                     struct rte_flow_error *error)
1151 {
1152         int ret;
1153         uint64_t action_flags = 0;
1154         uint64_t item_flags = 0;
1155         uint64_t last_item = 0;
1156         uint8_t next_protocol = 0xff;
1157         uint16_t ether_type = 0;
1158
1159         if (items == NULL)
1160                 return -1;
1161         ret = mlx5_flow_validate_attributes(dev, attr, error);
1162         if (ret < 0)
1163                 return ret;
1164         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1165                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1166                 int ret = 0;
1167
1168                 switch (items->type) {
1169                 case RTE_FLOW_ITEM_TYPE_VOID:
1170                         break;
1171                 case RTE_FLOW_ITEM_TYPE_ETH:
1172                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1173                                                           error);
1174                         if (ret < 0)
1175                                 return ret;
1176                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1177                                              MLX5_FLOW_LAYER_OUTER_L2;
1178                         if (items->mask != NULL && items->spec != NULL) {
1179                                 ether_type =
1180                                         ((const struct rte_flow_item_eth *)
1181                                          items->spec)->type;
1182                                 ether_type &=
1183                                         ((const struct rte_flow_item_eth *)
1184                                          items->mask)->type;
1185                                 ether_type = rte_be_to_cpu_16(ether_type);
1186                         } else {
1187                                 ether_type = 0;
1188                         }
1189                         break;
1190                 case RTE_FLOW_ITEM_TYPE_VLAN:
1191                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1192                                                            dev, error);
1193                         if (ret < 0)
1194                                 return ret;
1195                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1196                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1197                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1198                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1199                         if (items->mask != NULL && items->spec != NULL) {
1200                                 ether_type =
1201                                         ((const struct rte_flow_item_vlan *)
1202                                          items->spec)->inner_type;
1203                                 ether_type &=
1204                                         ((const struct rte_flow_item_vlan *)
1205                                          items->mask)->inner_type;
1206                                 ether_type = rte_be_to_cpu_16(ether_type);
1207                         } else {
1208                                 ether_type = 0;
1209                         }
1210                         break;
1211                 case RTE_FLOW_ITEM_TYPE_IPV4:
1212                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1213                                                            last_item,
1214                                                            ether_type, NULL,
1215                                                            error);
1216                         if (ret < 0)
1217                                 return ret;
1218                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1219                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1220                         if (items->mask != NULL &&
1221                             ((const struct rte_flow_item_ipv4 *)
1222                              items->mask)->hdr.next_proto_id) {
1223                                 next_protocol =
1224                                         ((const struct rte_flow_item_ipv4 *)
1225                                          (items->spec))->hdr.next_proto_id;
1226                                 next_protocol &=
1227                                         ((const struct rte_flow_item_ipv4 *)
1228                                          (items->mask))->hdr.next_proto_id;
1229                         } else {
1230                                 /* Reset for inner layer. */
1231                                 next_protocol = 0xff;
1232                         }
1233                         break;
1234                 case RTE_FLOW_ITEM_TYPE_IPV6:
1235                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1236                                                            last_item,
1237                                                            ether_type, NULL,
1238                                                            error);
1239                         if (ret < 0)
1240                                 return ret;
1241                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1242                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1243                         if (items->mask != NULL &&
1244                             ((const struct rte_flow_item_ipv6 *)
1245                              items->mask)->hdr.proto) {
1246                                 next_protocol =
1247                                         ((const struct rte_flow_item_ipv6 *)
1248                                          items->spec)->hdr.proto;
1249                                 next_protocol &=
1250                                         ((const struct rte_flow_item_ipv6 *)
1251                                          items->mask)->hdr.proto;
1252                         } else {
1253                                 /* Reset for inner layer. */
1254                                 next_protocol = 0xff;
1255                         }
1256                         break;
1257                 case RTE_FLOW_ITEM_TYPE_UDP:
1258                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1259                                                           next_protocol,
1260                                                           error);
1261                         if (ret < 0)
1262                                 return ret;
1263                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1264                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1265                         break;
1266                 case RTE_FLOW_ITEM_TYPE_TCP:
1267                         ret = mlx5_flow_validate_item_tcp
1268                                                 (items, item_flags,
1269                                                  next_protocol,
1270                                                  &rte_flow_item_tcp_mask,
1271                                                  error);
1272                         if (ret < 0)
1273                                 return ret;
1274                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1275                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1276                         break;
1277                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1278                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1279                                                             error);
1280                         if (ret < 0)
1281                                 return ret;
1282                         last_item = MLX5_FLOW_LAYER_VXLAN;
1283                         break;
1284                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1285                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1286                                                                 item_flags,
1287                                                                 dev, error);
1288                         if (ret < 0)
1289                                 return ret;
1290                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1291                         break;
1292                 case RTE_FLOW_ITEM_TYPE_GRE:
1293                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1294                                                           next_protocol, error);
1295                         if (ret < 0)
1296                                 return ret;
1297                         last_item = MLX5_FLOW_LAYER_GRE;
1298                         break;
1299                 case RTE_FLOW_ITEM_TYPE_MPLS:
1300                         ret = mlx5_flow_validate_item_mpls(dev, items,
1301                                                            item_flags,
1302                                                            last_item, error);
1303                         if (ret < 0)
1304                                 return ret;
1305                         last_item = MLX5_FLOW_LAYER_MPLS;
1306                         break;
1307                 default:
1308                         return rte_flow_error_set(error, ENOTSUP,
1309                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1310                                                   NULL, "item not supported");
1311                 }
1312                 item_flags |= last_item;
1313         }
1314         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1315                 switch (actions->type) {
1316                 case RTE_FLOW_ACTION_TYPE_VOID:
1317                         break;
1318                 case RTE_FLOW_ACTION_TYPE_FLAG:
1319                         ret = mlx5_flow_validate_action_flag(action_flags,
1320                                                              attr,
1321                                                              error);
1322                         if (ret < 0)
1323                                 return ret;
1324                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1325                         break;
1326                 case RTE_FLOW_ACTION_TYPE_MARK:
1327                         ret = mlx5_flow_validate_action_mark(actions,
1328                                                              action_flags,
1329                                                              attr,
1330                                                              error);
1331                         if (ret < 0)
1332                                 return ret;
1333                         action_flags |= MLX5_FLOW_ACTION_MARK;
1334                         break;
1335                 case RTE_FLOW_ACTION_TYPE_DROP:
1336                         ret = mlx5_flow_validate_action_drop(action_flags,
1337                                                              attr,
1338                                                              error);
1339                         if (ret < 0)
1340                                 return ret;
1341                         action_flags |= MLX5_FLOW_ACTION_DROP;
1342                         break;
1343                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1344                         ret = mlx5_flow_validate_action_queue(actions,
1345                                                               action_flags, dev,
1346                                                               attr,
1347                                                               error);
1348                         if (ret < 0)
1349                                 return ret;
1350                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1351                         break;
1352                 case RTE_FLOW_ACTION_TYPE_RSS:
1353                         ret = mlx5_flow_validate_action_rss(actions,
1354                                                             action_flags, dev,
1355                                                             attr, item_flags,
1356                                                             error);
1357                         if (ret < 0)
1358                                 return ret;
1359                         action_flags |= MLX5_FLOW_ACTION_RSS;
1360                         break;
1361                 case RTE_FLOW_ACTION_TYPE_COUNT:
1362                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1363                         if (ret < 0)
1364                                 return ret;
1365                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1366                         break;
1367                 default:
1368                         return rte_flow_error_set(error, ENOTSUP,
1369                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1370                                                   actions,
1371                                                   "action not supported");
1372                 }
1373         }
1374         /*
1375          * Validate the drop action mutual exclusion with other actions.
1376          * Drop action is mutually-exclusive with any other action, except for
1377          * Count action.
1378          */
1379         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1380             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1381                 return rte_flow_error_set(error, EINVAL,
1382                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1383                                           "Drop action is mutually-exclusive "
1384                                           "with any other action, except for "
1385                                           "Count action");
1386         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1387                 return rte_flow_error_set(error, EINVAL,
1388                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1389                                           "no fate action is found");
1390         return 0;
1391 }
1392
1393 /**
1394  * Calculate the required bytes that are needed for the action part of the verbs
1395  * flow.
1396  *
1397  * @param[in] actions
1398  *   Pointer to the list of actions.
1399  *
1400  * @return
1401  *   The size of the memory needed for all actions.
1402  */
1403 static int
1404 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1405 {
1406         int size = 0;
1407
1408         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1409                 switch (actions->type) {
1410                 case RTE_FLOW_ACTION_TYPE_VOID:
1411                         break;
1412                 case RTE_FLOW_ACTION_TYPE_FLAG:
1413                         size += sizeof(struct ibv_flow_spec_action_tag);
1414                         break;
1415                 case RTE_FLOW_ACTION_TYPE_MARK:
1416                         size += sizeof(struct ibv_flow_spec_action_tag);
1417                         break;
1418                 case RTE_FLOW_ACTION_TYPE_DROP:
1419                         size += sizeof(struct ibv_flow_spec_action_drop);
1420                         break;
1421                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1422                         break;
1423                 case RTE_FLOW_ACTION_TYPE_RSS:
1424                         break;
1425                 case RTE_FLOW_ACTION_TYPE_COUNT:
1426 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1427         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1428                         size += sizeof(struct ibv_flow_spec_counter_action);
1429 #endif
1430                         break;
1431                 default:
1432                         break;
1433                 }
1434         }
1435         return size;
1436 }
1437
1438 /**
1439  * Calculate the required bytes that are needed for the item part of the verbs
1440  * flow.
1441  *
1442  * @param[in] items
1443  *   Pointer to the list of items.
1444  *
1445  * @return
1446  *   The size of the memory needed for all items.
1447  */
1448 static int
1449 flow_verbs_get_items_size(const struct rte_flow_item items[])
1450 {
1451         int size = 0;
1452
1453         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1454                 switch (items->type) {
1455                 case RTE_FLOW_ITEM_TYPE_VOID:
1456                         break;
1457                 case RTE_FLOW_ITEM_TYPE_ETH:
1458                         size += sizeof(struct ibv_flow_spec_eth);
1459                         break;
1460                 case RTE_FLOW_ITEM_TYPE_VLAN:
1461                         size += sizeof(struct ibv_flow_spec_eth);
1462                         break;
1463                 case RTE_FLOW_ITEM_TYPE_IPV4:
1464                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1465                         break;
1466                 case RTE_FLOW_ITEM_TYPE_IPV6:
1467                         size += sizeof(struct ibv_flow_spec_ipv6);
1468                         break;
1469                 case RTE_FLOW_ITEM_TYPE_UDP:
1470                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1471                         break;
1472                 case RTE_FLOW_ITEM_TYPE_TCP:
1473                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1474                         break;
1475                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1476                         size += sizeof(struct ibv_flow_spec_tunnel);
1477                         break;
1478                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1479                         size += sizeof(struct ibv_flow_spec_tunnel);
1480                         break;
1481 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1482                 case RTE_FLOW_ITEM_TYPE_GRE:
1483                         size += sizeof(struct ibv_flow_spec_gre);
1484                         break;
1485                 case RTE_FLOW_ITEM_TYPE_MPLS:
1486                         size += sizeof(struct ibv_flow_spec_mpls);
1487                         break;
1488 #else
1489                 case RTE_FLOW_ITEM_TYPE_GRE:
1490                         size += sizeof(struct ibv_flow_spec_tunnel);
1491                         break;
1492 #endif
1493                 default:
1494                         break;
1495                 }
1496         }
1497         return size;
1498 }
1499
1500 /**
1501  * Internal preparation function. Allocate mlx5_flow with the required size.
1502  * The required size is calculate based on the actions and items. This function
1503  * also returns the detected actions and items for later use.
1504  *
1505  * @param[in] dev
1506  *   Pointer to Ethernet device.
1507  * @param[in] attr
1508  *   Pointer to the flow attributes.
1509  * @param[in] items
1510  *   Pointer to the list of items.
1511  * @param[in] actions
1512  *   Pointer to the list of actions.
1513  * @param[out] error
1514  *   Pointer to the error structure.
1515  *
1516  * @return
1517  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1518  *   is set.
1519  */
1520 static struct mlx5_flow *
1521 flow_verbs_prepare(struct rte_eth_dev *dev,
1522                    const struct rte_flow_attr *attr __rte_unused,
1523                    const struct rte_flow_item items[],
1524                    const struct rte_flow_action actions[],
1525                    struct rte_flow_error *error)
1526 {
1527         size_t size = 0;
1528         uint32_t handle_idx = 0;
1529         struct mlx5_flow *dev_flow;
1530         struct mlx5_flow_handle *dev_handle;
1531         struct mlx5_priv *priv = dev->data->dev_private;
1532
1533         size += flow_verbs_get_actions_size(actions);
1534         size += flow_verbs_get_items_size(items);
1535         if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1536                 rte_flow_error_set(error, E2BIG,
1537                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1538                                    "Verbs spec/action size too large");
1539                 return NULL;
1540         }
1541         /* In case of corrupting the memory. */
1542         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1543                 rte_flow_error_set(error, ENOSPC,
1544                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1545                                    "not free temporary device flow");
1546                 return NULL;
1547         }
1548         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1549                                    &handle_idx);
1550         if (!dev_handle) {
1551                 rte_flow_error_set(error, ENOMEM,
1552                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1553                                    "not enough memory to create flow handle");
1554                 return NULL;
1555         }
1556         /* No multi-thread supporting. */
1557         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
1558         dev_flow->handle = dev_handle;
1559         dev_flow->handle_idx = handle_idx;
1560         /* Memcpy is used, only size needs to be cleared to 0. */
1561         dev_flow->verbs.size = 0;
1562         dev_flow->verbs.attr.num_of_specs = 0;
1563         dev_flow->ingress = attr->ingress;
1564         dev_flow->hash_fields = 0;
1565         /* Need to set transfer attribute: not supported in Verbs mode. */
1566         return dev_flow;
1567 }
1568
1569 /**
1570  * Fill the flow with verb spec.
1571  *
1572  * @param[in] dev
1573  *   Pointer to Ethernet device.
1574  * @param[in, out] dev_flow
1575  *   Pointer to the mlx5 flow.
1576  * @param[in] attr
1577  *   Pointer to the flow attributes.
1578  * @param[in] items
1579  *   Pointer to the list of items.
1580  * @param[in] actions
1581  *   Pointer to the list of actions.
1582  * @param[out] error
1583  *   Pointer to the error structure.
1584  *
1585  * @return
1586  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1587  */
1588 static int
1589 flow_verbs_translate(struct rte_eth_dev *dev,
1590                      struct mlx5_flow *dev_flow,
1591                      const struct rte_flow_attr *attr,
1592                      const struct rte_flow_item items[],
1593                      const struct rte_flow_action actions[],
1594                      struct rte_flow_error *error)
1595 {
1596         uint64_t item_flags = 0;
1597         uint64_t action_flags = 0;
1598         uint64_t priority = attr->priority;
1599         uint32_t subpriority = 0;
1600         struct mlx5_priv *priv = dev->data->dev_private;
1601         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
1602                                               priv->rss_desc)
1603                                               [!!priv->flow_nested_idx];
1604
1605         if (priority == MLX5_FLOW_PRIO_RSVD)
1606                 priority = priv->config.flow_prio - 1;
1607         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1608                 int ret;
1609
1610                 switch (actions->type) {
1611                 case RTE_FLOW_ACTION_TYPE_VOID:
1612                         break;
1613                 case RTE_FLOW_ACTION_TYPE_FLAG:
1614                         flow_verbs_translate_action_flag(dev_flow, actions);
1615                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1616                         dev_flow->handle->mark = 1;
1617                         break;
1618                 case RTE_FLOW_ACTION_TYPE_MARK:
1619                         flow_verbs_translate_action_mark(dev_flow, actions);
1620                         action_flags |= MLX5_FLOW_ACTION_MARK;
1621                         dev_flow->handle->mark = 1;
1622                         break;
1623                 case RTE_FLOW_ACTION_TYPE_DROP:
1624                         flow_verbs_translate_action_drop(dev_flow, actions);
1625                         action_flags |= MLX5_FLOW_ACTION_DROP;
1626                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
1627                         break;
1628                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1629                         flow_verbs_translate_action_queue(rss_desc, actions);
1630                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1631                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1632                         break;
1633                 case RTE_FLOW_ACTION_TYPE_RSS:
1634                         flow_verbs_translate_action_rss(rss_desc, actions);
1635                         action_flags |= MLX5_FLOW_ACTION_RSS;
1636                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1637                         break;
1638                 case RTE_FLOW_ACTION_TYPE_COUNT:
1639                         ret = flow_verbs_translate_action_count(dev_flow,
1640                                                                 actions,
1641                                                                 dev, error);
1642                         if (ret < 0)
1643                                 return ret;
1644                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1645                         break;
1646                 default:
1647                         return rte_flow_error_set(error, ENOTSUP,
1648                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1649                                                   actions,
1650                                                   "action not supported");
1651                 }
1652         }
1653         dev_flow->act_flags = action_flags;
1654         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1655                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1656
1657                 switch (items->type) {
1658                 case RTE_FLOW_ITEM_TYPE_VOID:
1659                         break;
1660                 case RTE_FLOW_ITEM_TYPE_ETH:
1661                         flow_verbs_translate_item_eth(dev_flow, items,
1662                                                       item_flags);
1663                         subpriority = MLX5_PRIORITY_MAP_L2;
1664                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1665                                                MLX5_FLOW_LAYER_OUTER_L2;
1666                         break;
1667                 case RTE_FLOW_ITEM_TYPE_VLAN:
1668                         flow_verbs_translate_item_vlan(dev_flow, items,
1669                                                        item_flags);
1670                         subpriority = MLX5_PRIORITY_MAP_L2;
1671                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1672                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1673                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1674                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1675                         break;
1676                 case RTE_FLOW_ITEM_TYPE_IPV4:
1677                         flow_verbs_translate_item_ipv4(dev_flow, items,
1678                                                        item_flags);
1679                         subpriority = MLX5_PRIORITY_MAP_L3;
1680                         dev_flow->hash_fields |=
1681                                 mlx5_flow_hashfields_adjust
1682                                         (rss_desc, tunnel,
1683                                          MLX5_IPV4_LAYER_TYPES,
1684                                          MLX5_IPV4_IBV_RX_HASH);
1685                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1686                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1687                         break;
1688                 case RTE_FLOW_ITEM_TYPE_IPV6:
1689                         flow_verbs_translate_item_ipv6(dev_flow, items,
1690                                                        item_flags);
1691                         subpriority = MLX5_PRIORITY_MAP_L3;
1692                         dev_flow->hash_fields |=
1693                                 mlx5_flow_hashfields_adjust
1694                                         (rss_desc, tunnel,
1695                                          MLX5_IPV6_LAYER_TYPES,
1696                                          MLX5_IPV6_IBV_RX_HASH);
1697                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1698                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1699                         break;
1700                 case RTE_FLOW_ITEM_TYPE_TCP:
1701                         flow_verbs_translate_item_tcp(dev_flow, items,
1702                                                       item_flags);
1703                         subpriority = MLX5_PRIORITY_MAP_L4;
1704                         dev_flow->hash_fields |=
1705                                 mlx5_flow_hashfields_adjust
1706                                         (rss_desc, tunnel, ETH_RSS_TCP,
1707                                          (IBV_RX_HASH_SRC_PORT_TCP |
1708                                           IBV_RX_HASH_DST_PORT_TCP));
1709                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1710                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1711                         break;
1712                 case RTE_FLOW_ITEM_TYPE_UDP:
1713                         flow_verbs_translate_item_udp(dev_flow, items,
1714                                                       item_flags);
1715                         subpriority = MLX5_PRIORITY_MAP_L4;
1716                         dev_flow->hash_fields |=
1717                                 mlx5_flow_hashfields_adjust
1718                                         (rss_desc, tunnel, ETH_RSS_UDP,
1719                                          (IBV_RX_HASH_SRC_PORT_UDP |
1720                                           IBV_RX_HASH_DST_PORT_UDP));
1721                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1722                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1723                         break;
1724                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1725                         flow_verbs_translate_item_vxlan(dev_flow, items,
1726                                                         item_flags);
1727                         subpriority = MLX5_PRIORITY_MAP_L2;
1728                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1729                         break;
1730                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1731                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1732                                                             item_flags);
1733                         subpriority = MLX5_PRIORITY_MAP_L2;
1734                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1735                         break;
1736                 case RTE_FLOW_ITEM_TYPE_GRE:
1737                         flow_verbs_translate_item_gre(dev_flow, items,
1738                                                       item_flags);
1739                         subpriority = MLX5_PRIORITY_MAP_L2;
1740                         item_flags |= MLX5_FLOW_LAYER_GRE;
1741                         break;
1742                 case RTE_FLOW_ITEM_TYPE_MPLS:
1743                         flow_verbs_translate_item_mpls(dev_flow, items,
1744                                                        item_flags);
1745                         subpriority = MLX5_PRIORITY_MAP_L2;
1746                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1747                         break;
1748                 default:
1749                         return rte_flow_error_set(error, ENOTSUP,
1750                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1751                                                   NULL,
1752                                                   "item not supported");
1753                 }
1754         }
1755         dev_flow->handle->layers = item_flags;
1756         /* Other members of attr will be ignored. */
1757         dev_flow->verbs.attr.priority =
1758                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1759         dev_flow->verbs.attr.port = (uint8_t)priv->ibv_port;
1760         return 0;
1761 }
1762
1763 /**
1764  * Remove the flow from the NIC but keeps it in memory.
1765  *
1766  * @param[in] dev
1767  *   Pointer to the Ethernet device structure.
1768  * @param[in, out] flow
1769  *   Pointer to flow structure.
1770  */
1771 static void
1772 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1773 {
1774         struct mlx5_priv *priv = dev->data->dev_private;
1775         struct mlx5_flow_handle *handle;
1776         uint32_t handle_idx;
1777
1778         if (!flow)
1779                 return;
1780         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1781                        handle_idx, handle, next) {
1782                 if (handle->ib_flow) {
1783                         claim_zero(mlx5_glue->destroy_flow(handle->ib_flow));
1784                         handle->ib_flow = NULL;
1785                 }
1786                 /* hrxq is union, don't touch it only the flag is set. */
1787                 if (handle->rix_hrxq) {
1788                         if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1789                                 mlx5_hrxq_drop_release(dev);
1790                                 handle->rix_hrxq = 0;
1791                         } else if (handle->fate_action ==
1792                                    MLX5_FLOW_FATE_QUEUE) {
1793                                 mlx5_hrxq_release(dev, handle->rix_hrxq);
1794                                 handle->rix_hrxq = 0;
1795                         }
1796                 }
1797                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1798                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1799         }
1800 }
1801
1802 /**
1803  * Remove the flow from the NIC and the memory.
1804  *
1805  * @param[in] dev
1806  *   Pointer to the Ethernet device structure.
1807  * @param[in, out] flow
1808  *   Pointer to flow structure.
1809  */
1810 static void
1811 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1812 {
1813         struct mlx5_priv *priv = dev->data->dev_private;
1814         struct mlx5_flow_handle *handle;
1815
1816         if (!flow)
1817                 return;
1818         flow_verbs_remove(dev, flow);
1819         while (flow->dev_handles) {
1820                 uint32_t tmp_idx = flow->dev_handles;
1821
1822                 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1823                                    tmp_idx);
1824                 if (!handle)
1825                         return;
1826                 flow->dev_handles = handle->next.next;
1827                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1828                            tmp_idx);
1829         }
1830         if (flow->counter) {
1831                 flow_verbs_counter_release(dev, flow->counter);
1832                 flow->counter = 0;
1833         }
1834 }
1835
1836 /**
1837  * Apply the flow to the NIC.
1838  *
1839  * @param[in] dev
1840  *   Pointer to the Ethernet device structure.
1841  * @param[in, out] flow
1842  *   Pointer to flow structure.
1843  * @param[out] error
1844  *   Pointer to error structure.
1845  *
1846  * @return
1847  *   0 on success, a negative errno value otherwise and rte_errno is set.
1848  */
1849 static int
1850 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1851                  struct rte_flow_error *error)
1852 {
1853         struct mlx5_priv *priv = dev->data->dev_private;
1854         struct mlx5_flow_handle *handle;
1855         struct mlx5_flow *dev_flow;
1856         struct mlx5_hrxq *hrxq;
1857         uint32_t dev_handles;
1858         int err;
1859         int idx;
1860
1861         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
1862                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
1863                 handle = dev_flow->handle;
1864                 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1865                         hrxq = mlx5_hrxq_drop_new(dev);
1866                         if (!hrxq) {
1867                                 rte_flow_error_set
1868                                         (error, errno,
1869                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1870                                          "cannot get drop hash queue");
1871                                 goto error;
1872                         }
1873                 } else {
1874                         uint32_t hrxq_idx;
1875                         struct mlx5_flow_rss_desc *rss_desc =
1876                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
1877                                 [!!priv->flow_nested_idx];
1878
1879                         MLX5_ASSERT(rss_desc->queue_num);
1880                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
1881                                              MLX5_RSS_HASH_KEY_LEN,
1882                                              dev_flow->hash_fields,
1883                                              rss_desc->queue,
1884                                              rss_desc->queue_num);
1885                         if (!hrxq_idx)
1886                                 hrxq_idx = mlx5_hrxq_new(dev, rss_desc->key,
1887                                                 MLX5_RSS_HASH_KEY_LEN,
1888                                                 dev_flow->hash_fields,
1889                                                 rss_desc->queue,
1890                                                 rss_desc->queue_num,
1891                                                 !!(handle->layers &
1892                                                 MLX5_FLOW_LAYER_TUNNEL));
1893                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1894                                          hrxq_idx);
1895                         if (!hrxq) {
1896                                 rte_flow_error_set
1897                                         (error, rte_errno,
1898                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1899                                          "cannot get hash queue");
1900                                 goto error;
1901                         }
1902                         handle->rix_hrxq = hrxq_idx;
1903                 }
1904                 MLX5_ASSERT(hrxq);
1905                 handle->ib_flow = mlx5_glue->create_flow(hrxq->qp,
1906                                                      &dev_flow->verbs.attr);
1907                 if (!handle->ib_flow) {
1908                         rte_flow_error_set(error, errno,
1909                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1910                                            NULL,
1911                                            "hardware refuses to create flow");
1912                         goto error;
1913                 }
1914                 if (priv->vmwa_context &&
1915                     handle->vf_vlan.tag && !handle->vf_vlan.created) {
1916                         /*
1917                          * The rule contains the VLAN pattern.
1918                          * For VF we are going to create VLAN
1919                          * interface to make hypervisor set correct
1920                          * e-Switch vport context.
1921                          */
1922                         mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
1923                 }
1924         }
1925         return 0;
1926 error:
1927         err = rte_errno; /* Save rte_errno before cleanup. */
1928         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1929                        dev_handles, handle, next) {
1930                 /* hrxq is union, don't touch it only the flag is set. */
1931                 if (handle->rix_hrxq) {
1932                         if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1933                                 mlx5_hrxq_drop_release(dev);
1934                                 handle->rix_hrxq = 0;
1935                         } else if (handle->fate_action ==
1936                                    MLX5_FLOW_FATE_QUEUE) {
1937                                 mlx5_hrxq_release(dev, handle->rix_hrxq);
1938                                 handle->rix_hrxq = 0;
1939                         }
1940                 }
1941                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1942                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1943         }
1944         rte_errno = err; /* Restore rte_errno. */
1945         return -rte_errno;
1946 }
1947
1948 /**
1949  * Query a flow.
1950  *
1951  * @see rte_flow_query()
1952  * @see rte_flow_ops
1953  */
1954 static int
1955 flow_verbs_query(struct rte_eth_dev *dev,
1956                  struct rte_flow *flow,
1957                  const struct rte_flow_action *actions,
1958                  void *data,
1959                  struct rte_flow_error *error)
1960 {
1961         int ret = -EINVAL;
1962
1963         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1964                 switch (actions->type) {
1965                 case RTE_FLOW_ACTION_TYPE_VOID:
1966                         break;
1967                 case RTE_FLOW_ACTION_TYPE_COUNT:
1968                         ret = flow_verbs_counter_query(dev, flow, data, error);
1969                         break;
1970                 default:
1971                         return rte_flow_error_set(error, ENOTSUP,
1972                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1973                                                   actions,
1974                                                   "action not supported");
1975                 }
1976         }
1977         return ret;
1978 }
1979
1980 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1981         .validate = flow_verbs_validate,
1982         .prepare = flow_verbs_prepare,
1983         .translate = flow_verbs_translate,
1984         .apply = flow_verbs_apply,
1985         .remove = flow_verbs_remove,
1986         .destroy = flow_verbs_destroy,
1987         .query = flow_verbs_query,
1988 };