drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_ip.h>
  18
  19 #include <mlx5_glue.h>
  20 #include <mlx5_prm.h>
  21 #include <mlx5_malloc.h>
  22
  23 #include "mlx5_defs.h"
  24 #include "mlx5.h"
  25 #include "mlx5_flow.h"
  26 #include "mlx5_rx.h"
  27
  28 #define VERBS_SPEC_INNER(item_flags) \
  29         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  30
  31 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
  32 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
  33         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
  34 };
  35
  36 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
  37 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
  38         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
  39         { 9, 10, 11 }, { 12, 13, 14 },
  40 };
  41
  42 /* Verbs specification header. */
  43 struct ibv_spec_header {
  44         enum ibv_flow_spec_type type;
  45         uint16_t size;
  46 };
  47
  48 /**
  49  * Discover the maximum number of priority available.
  50  *
  51  * @param[in] dev
  52  *   Pointer to the Ethernet device structure.
  53  *
  54  * @return
  55  *   number of supported flow priority on success, a negative errno
  56  *   value otherwise and rte_errno is set.
  57  */
  58 int
  59 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
  60 {
  61         struct mlx5_priv *priv = dev->data->dev_private;
  62         struct {
  63                 struct ibv_flow_attr attr;
  64                 struct ibv_flow_spec_eth eth;
  65                 struct ibv_flow_spec_action_drop drop;
  66         } flow_attr = {
  67                 .attr = {
  68                         .num_of_specs = 2,
  69                         .port = (uint8_t)priv->dev_port,
  70                 },
  71                 .eth = {
  72                         .type = IBV_FLOW_SPEC_ETH,
  73                         .size = sizeof(struct ibv_flow_spec_eth),
  74                 },
  75                 .drop = {
  76                         .size = sizeof(struct ibv_flow_spec_action_drop),
  77                         .type = IBV_FLOW_SPEC_ACTION_DROP,
  78                 },
  79         };
  80         struct ibv_flow *flow;
  81         struct mlx5_hrxq *drop = priv->drop_queue.hrxq;
  82         uint16_t vprio[] = { 8, 16 };
  83         int i;
  84         int priority = 0;
  85
  86         if (!drop->qp) {
  87                 rte_errno = ENOTSUP;
  88                 return -rte_errno;
  89         }
  90         for (i = 0; i != RTE_DIM(vprio); i++) {
  91                 flow_attr.attr.priority = vprio[i] - 1;
  92                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
  93                 if (!flow)
  94                         break;
  95                 claim_zero(mlx5_glue->destroy_flow(flow));
  96                 priority = vprio[i];
  97         }
  98         switch (priority) {
  99         case 8:
 100                 priority = RTE_DIM(priority_map_3);
 101                 break;
 102         case 16:
 103                 priority = RTE_DIM(priority_map_5);
 104                 break;
 105         default:
 106                 rte_errno = ENOTSUP;
 107                 DRV_LOG(ERR,
 108                         "port %u verbs maximum priority: %d expected 8/16",
 109                         dev->data->port_id, priority);
 110                 return -rte_errno;
 111         }
 112         DRV_LOG(INFO, "port %u supported flow priorities:"
 113                 " 0-%d for ingress or egress root table,"
 114                 " 0-%d for non-root table or transfer root table.",
 115                 dev->data->port_id, priority - 2,
 116                 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1);
 117         return priority;
 118 }
 119
 120 /**
 121  * Adjust flow priority based on the highest layer and the request priority.
 122  *
 123  * @param[in] dev
 124  *   Pointer to the Ethernet device structure.
 125  * @param[in] priority
 126  *   The rule base priority.
 127  * @param[in] subpriority
 128  *   The priority based on the items.
 129  *
 130  * @return
 131  *   The new priority.
 132  */
 133 uint32_t
 134 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 135                                    uint32_t subpriority)
 136 {
 137         uint32_t res = 0;
 138         struct mlx5_priv *priv = dev->data->dev_private;
 139
 140         switch (priv->config.flow_prio) {
 141         case RTE_DIM(priority_map_3):
 142                 res = priority_map_3[priority][subpriority];
 143                 break;
 144         case RTE_DIM(priority_map_5):
 145                 res = priority_map_5[priority][subpriority];
 146                 break;
 147         }
 148         return  res;
 149 }
 150
 151 /**
 152  * Get Verbs flow counter by index.
 153  *
 154  * @param[in] dev
 155  *   Pointer to the Ethernet device structure.
 156  * @param[in] idx
 157  *   mlx5 flow counter index in the container.
 158  * @param[out] ppool
 159  *   mlx5 flow counter pool in the container,
 160  *
 161  * @return
 162  *   A pointer to the counter, NULL otherwise.
 163  */
 164 static struct mlx5_flow_counter *
 165 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
 166                               uint32_t idx,
 167                               struct mlx5_flow_counter_pool **ppool)
 168 {
 169         struct mlx5_priv *priv = dev->data->dev_private;
 170         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 171         struct mlx5_flow_counter_pool *pool;
 172
 173         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
 174         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
 175         MLX5_ASSERT(pool);
 176         if (ppool)
 177                 *ppool = pool;
 178         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
 179 }
 180
 181 /**
 182  * Create Verbs flow counter with Verbs library.
 183  *
 184  * @param[in] dev
 185  *   Pointer to the Ethernet device structure.
 186  * @param[in, out] counter
 187  *   mlx5 flow counter object, contains the counter id,
 188  *   handle of created Verbs flow counter is returned
 189  *   in cs field (if counters are supported).
 190  *
 191  * @return
 192  *   0 On success else a negative errno value is returned
 193  *   and rte_errno is set.
 194  */
 195 static int
 196 flow_verbs_counter_create(struct rte_eth_dev *dev,
 197                           struct mlx5_flow_counter *counter)
 198 {
 199 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 200         struct mlx5_priv *priv = dev->data->dev_private;
 201         struct ibv_context *ctx = priv->sh->ctx;
 202         struct ibv_counter_set_init_attr init = {
 203                          .counter_set_id = counter->shared_info.id};
 204
 205         counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init);
 206         if (!counter->dcs_when_free) {
 207                 rte_errno = ENOTSUP;
 208                 return -ENOTSUP;
 209         }
 210         return 0;
 211 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 212         struct mlx5_priv *priv = dev->data->dev_private;
 213         struct ibv_context *ctx = priv->sh->ctx;
 214         struct ibv_counters_init_attr init = {0};
 215         struct ibv_counter_attach_attr attach;
 216         int ret;
 217
 218         memset(&attach, 0, sizeof(attach));
 219         counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init);
 220         if (!counter->dcs_when_free) {
 221                 rte_errno = ENOTSUP;
 222                 return -ENOTSUP;
 223         }
 224         attach.counter_desc = IBV_COUNTER_PACKETS;
 225         attach.index = 0;
 226         ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL);
 227         if (!ret) {
 228                 attach.counter_desc = IBV_COUNTER_BYTES;
 229                 attach.index = 1;
 230                 ret = mlx5_glue->attach_counters
 231                                         (counter->dcs_when_free, &attach, NULL);
 232         }
 233         if (ret) {
 234                 claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free));
 235                 counter->dcs_when_free = NULL;
 236                 rte_errno = ret;
 237                 return -ret;
 238         }
 239         return 0;
 240 #else
 241         (void)dev;
 242         (void)counter;
 243         rte_errno = ENOTSUP;
 244         return -ENOTSUP;
 245 #endif
 246 }
 247
 248 /**
 249  * Get a flow counter.
 250  *
 251  * @param[in] dev
 252  *   Pointer to the Ethernet device structure.
 253  * @param[in] shared
 254  *   Indicate if this counter is shared with other flows.
 255  * @param[in] id
 256  *   Counter identifier.
 257  *
 258  * @return
 259  *   Index to the counter, 0 otherwise and rte_errno is set.
 260  */
 261 static uint32_t
 262 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 263 {
 264         struct mlx5_priv *priv = dev->data->dev_private;
 265         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 266         struct mlx5_flow_counter_pool *pool = NULL;
 267         struct mlx5_flow_counter *cnt = NULL;
 268         union mlx5_l3t_data data;
 269         uint32_t n_valid = cmng->n_valid;
 270         uint32_t pool_idx, cnt_idx;
 271         uint32_t i;
 272         int ret;
 273
 274         if (shared && !mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) &&
 275             data.dword)
 276                 return data.dword;
 277         for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 278                 pool = cmng->pools[pool_idx];
 279                 if (!pool)
 280                         continue;
 281                 cnt = TAILQ_FIRST(&pool->counters[0]);
 282                 if (cnt)
 283                         break;
 284         }
 285         if (!cnt) {
 286                 struct mlx5_flow_counter_pool **pools;
 287                 uint32_t size;
 288
 289                 if (n_valid == cmng->n) {
 290                         /* Resize the container pool array. */
 291                         size = sizeof(struct mlx5_flow_counter_pool *) *
 292                                      (n_valid + MLX5_CNT_CONTAINER_RESIZE);
 293                         pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0,
 294                                             SOCKET_ID_ANY);
 295                         if (!pools)
 296                                 return 0;
 297                         if (n_valid) {
 298                                 memcpy(pools, cmng->pools,
 299                                        sizeof(struct mlx5_flow_counter_pool *) *
 300                                        n_valid);
 301                                 mlx5_free(cmng->pools);
 302                         }
 303                         cmng->pools = pools;
 304                         cmng->n += MLX5_CNT_CONTAINER_RESIZE;
 305                 }
 306                 /* Allocate memory for new pool*/
 307                 size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL;
 308                 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
 309                 if (!pool)
 310                         return 0;
 311                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
 312                         cnt = MLX5_POOL_GET_CNT(pool, i);
 313                         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 314                 }
 315                 cnt = MLX5_POOL_GET_CNT(pool, 0);
 316                 cmng->pools[n_valid] = pool;
 317                 pool_idx = n_valid;
 318                 cmng->n_valid++;
 319         }
 320         TAILQ_REMOVE(&pool->counters[0], cnt, next);
 321         i = MLX5_CNT_ARRAY_IDX(pool, cnt);
 322         cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i);
 323         if (shared) {
 324                 data.dword = cnt_idx;
 325                 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
 326                         return 0;
 327                 cnt->shared_info.id = id;
 328                 cnt_idx |= MLX5_CNT_SHARED_OFFSET;
 329         }
 330         /* Create counter with Verbs. */
 331         ret = flow_verbs_counter_create(dev, cnt);
 332         if (!ret) {
 333                 cnt->dcs_when_active = cnt->dcs_when_free;
 334                 cnt->hits = 0;
 335                 cnt->bytes = 0;
 336                 return cnt_idx;
 337         }
 338         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 339         /* Some error occurred in Verbs library. */
 340         rte_errno = -ret;
 341         return 0;
 342 }
 343
 344 /**
 345  * Release a flow counter.
 346  *
 347  * @param[in] dev
 348  *   Pointer to the Ethernet device structure.
 349  * @param[in] counter
 350  *   Index to the counter handler.
 351  */
 352 static void
 353 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
 354 {
 355         struct mlx5_priv *priv = dev->data->dev_private;
 356         struct mlx5_flow_counter_pool *pool;
 357         struct mlx5_flow_counter *cnt;
 358
 359         cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool);
 360         if (IS_LEGACY_SHARED_CNT(counter) &&
 361             mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
 362                 return;
 363 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 364         claim_zero(mlx5_glue->destroy_counter_set
 365                         ((struct ibv_counter_set *)cnt->dcs_when_active));
 366 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 367         claim_zero(mlx5_glue->destroy_counters
 368                                 ((struct ibv_counters *)cnt->dcs_when_active));
 369 #endif
 370         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 371 }
 372
 373 /**
 374  * Query a flow counter via Verbs library call.
 375  *
 376  * @see rte_flow_query()
 377  * @see rte_flow_ops
 378  */
 379 static int
 380 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 381                          struct rte_flow *flow, void *data,
 382                          struct rte_flow_error *error)
 383 {
 384 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 385         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 386         if (flow->counter) {
 387                 struct mlx5_flow_counter_pool *pool;
 388                 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
 389                                                 (dev, flow->counter, &pool);
 390                 struct rte_flow_query_count *qc = data;
 391                 uint64_t counters[2] = {0, 0};
 392 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 393                 struct ibv_query_counter_set_attr query_cs_attr = {
 394                         .dcs_when_free = (struct ibv_counter_set *)
 395                                                 cnt->dcs_when_active,
 396                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 397                 };
 398                 struct ibv_counter_set_data query_out = {
 399                         .out = counters,
 400                         .outlen = 2 * sizeof(uint64_t),
 401                 };
 402                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 403                                                        &query_out);
 404 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 405                 int err = mlx5_glue->query_counters
 406                         ((struct ibv_counters *)cnt->dcs_when_active, counters,
 407                                 RTE_DIM(counters),
 408                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 409 #endif
 410                 if (err)
 411                         return rte_flow_error_set
 412                                 (error, err,
 413                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 414                                  NULL,
 415                                  "cannot read counter");
 416                 qc->hits_set = 1;
 417                 qc->bytes_set = 1;
 418                 qc->hits = counters[0] - cnt->hits;
 419                 qc->bytes = counters[1] - cnt->bytes;
 420                 if (qc->reset) {
 421                         cnt->hits = counters[0];
 422                         cnt->bytes = counters[1];
 423                 }
 424                 return 0;
 425         }
 426         return rte_flow_error_set(error, EINVAL,
 427                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 428                                   NULL,
 429                                   "flow does not have counter");
 430 #else
 431         (void)flow;
 432         (void)data;
 433         return rte_flow_error_set(error, ENOTSUP,
 434                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 435                                   NULL,
 436                                   "counters are not available");
 437 #endif
 438 }
 439
 440 /**
 441  * Add a verbs item specification into @p verbs.
 442  *
 443  * @param[out] verbs
 444  *   Pointer to verbs structure.
 445  * @param[in] src
 446  *   Create specification.
 447  * @param[in] size
 448  *   Size in bytes of the specification to copy.
 449  */
 450 static void
 451 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
 452                     void *src, unsigned int size)
 453 {
 454         void *dst;
 455
 456         if (!verbs)
 457                 return;
 458         MLX5_ASSERT(verbs->specs);
 459         dst = (void *)(verbs->specs + verbs->size);
 460         memcpy(dst, src, size);
 461         ++verbs->attr.num_of_specs;
 462         verbs->size += size;
 463 }
 464
 465 /**
 466  * Convert the @p item into a Verbs specification. This function assumes that
 467  * the input is valid and that there is space to insert the requested item
 468  * into the flow.
 469  *
 470  * @param[in, out] dev_flow
 471  *   Pointer to dev_flow structure.
 472  * @param[in] item
 473  *   Item specification.
 474  * @param[in] item_flags
 475  *   Parsed item flags.
 476  */
 477 static void
 478 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 479                               const struct rte_flow_item *item,
 480                               uint64_t item_flags)
 481 {
 482         const struct rte_flow_item_eth *spec = item->spec;
 483         const struct rte_flow_item_eth *mask = item->mask;
 484         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 485         struct ibv_flow_spec_eth eth = {
 486                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 487                 .size = size,
 488         };
 489
 490         if (!mask)
 491                 mask = &rte_flow_item_eth_mask;
 492         if (spec) {
 493                 unsigned int i;
 494
 495                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 496                         RTE_ETHER_ADDR_LEN);
 497                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 498                         RTE_ETHER_ADDR_LEN);
 499                 eth.val.ether_type = spec->type;
 500                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 501                         RTE_ETHER_ADDR_LEN);
 502                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 503                         RTE_ETHER_ADDR_LEN);
 504                 eth.mask.ether_type = mask->type;
 505                 /* Remove unwanted bits from values. */
 506                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 507                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 508                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 509                 }
 510                 eth.val.ether_type &= eth.mask.ether_type;
 511         }
 512         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 513 }
 514
 515 /**
 516  * Update the VLAN tag in the Verbs Ethernet specification.
 517  * This function assumes that the input is valid and there is space to add
 518  * the requested item.
 519  *
 520  * @param[in, out] attr
 521  *   Pointer to Verbs attributes structure.
 522  * @param[in] eth
 523  *   Verbs structure containing the VLAN information to copy.
 524  */
 525 static void
 526 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 527                             struct ibv_flow_spec_eth *eth)
 528 {
 529         unsigned int i;
 530         const enum ibv_flow_spec_type search = eth->type;
 531         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 532                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 533
 534         for (i = 0; i != attr->num_of_specs; ++i) {
 535                 if (hdr->type == search) {
 536                         struct ibv_flow_spec_eth *e =
 537                                 (struct ibv_flow_spec_eth *)hdr;
 538
 539                         e->val.vlan_tag = eth->val.vlan_tag;
 540                         e->mask.vlan_tag = eth->mask.vlan_tag;
 541                         e->val.ether_type = eth->val.ether_type;
 542                         e->mask.ether_type = eth->mask.ether_type;
 543                         break;
 544                 }
 545                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 546         }
 547 }
 548
 549 /**
 550  * Convert the @p item into a Verbs specification. This function assumes that
 551  * the input is valid and that there is space to insert the requested item
 552  * into the flow.
 553  *
 554  * @param[in, out] dev_flow
 555  *   Pointer to dev_flow structure.
 556  * @param[in] item
 557  *   Item specification.
 558  * @param[in] item_flags
 559  *   Parsed item flags.
 560  */
 561 static void
 562 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 563                                const struct rte_flow_item *item,
 564                                uint64_t item_flags)
 565 {
 566         const struct rte_flow_item_vlan *spec = item->spec;
 567         const struct rte_flow_item_vlan *mask = item->mask;
 568         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 569         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 570         struct ibv_flow_spec_eth eth = {
 571                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 572                 .size = size,
 573         };
 574         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 575                                       MLX5_FLOW_LAYER_OUTER_L2;
 576
 577         if (!mask)
 578                 mask = &rte_flow_item_vlan_mask;
 579         if (spec) {
 580                 eth.val.vlan_tag = spec->tci;
 581                 eth.mask.vlan_tag = mask->tci;
 582                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 583                 eth.val.ether_type = spec->inner_type;
 584                 eth.mask.ether_type = mask->inner_type;
 585                 eth.val.ether_type &= eth.mask.ether_type;
 586         }
 587         if (!(item_flags & l2m))
 588                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 589         else
 590                 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, &eth);
 591         if (!tunnel)
 592                 dev_flow->handle->vf_vlan.tag =
 593                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 594 }
 595
 596 /**
 597  * Convert the @p item into a Verbs specification. This function assumes that
 598  * the input is valid and that there is space to insert the requested item
 599  * into the flow.
 600  *
 601  * @param[in, out] dev_flow
 602  *   Pointer to dev_flow structure.
 603  * @param[in] item
 604  *   Item specification.
 605  * @param[in] item_flags
 606  *   Parsed item flags.
 607  */
 608 static void
 609 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 610                                const struct rte_flow_item *item,
 611                                uint64_t item_flags)
 612 {
 613         const struct rte_flow_item_ipv4 *spec = item->spec;
 614         const struct rte_flow_item_ipv4 *mask = item->mask;
 615         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 616         struct ibv_flow_spec_ipv4_ext ipv4 = {
 617                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 618                 .size = size,
 619         };
 620
 621         if (!mask)
 622                 mask = &rte_flow_item_ipv4_mask;
 623         if (spec) {
 624                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 625                         .src_ip = spec->hdr.src_addr,
 626                         .dst_ip = spec->hdr.dst_addr,
 627                         .proto = spec->hdr.next_proto_id,
 628                         .tos = spec->hdr.type_of_service,
 629                 };
 630                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 631                         .src_ip = mask->hdr.src_addr,
 632                         .dst_ip = mask->hdr.dst_addr,
 633                         .proto = mask->hdr.next_proto_id,
 634                         .tos = mask->hdr.type_of_service,
 635                 };
 636                 /* Remove unwanted bits from values. */
 637                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 638                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 639                 ipv4.val.proto &= ipv4.mask.proto;
 640                 ipv4.val.tos &= ipv4.mask.tos;
 641         }
 642         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 643 }
 644
 645 /**
 646  * Convert the @p item into a Verbs specification. This function assumes that
 647  * the input is valid and that there is space to insert the requested item
 648  * into the flow.
 649  *
 650  * @param[in, out] dev_flow
 651  *   Pointer to dev_flow structure.
 652  * @param[in] item
 653  *   Item specification.
 654  * @param[in] item_flags
 655  *   Parsed item flags.
 656  */
 657 static void
 658 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 659                                const struct rte_flow_item *item,
 660                                uint64_t item_flags)
 661 {
 662         const struct rte_flow_item_ipv6 *spec = item->spec;
 663         const struct rte_flow_item_ipv6 *mask = item->mask;
 664         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 665         struct ibv_flow_spec_ipv6 ipv6 = {
 666                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 667                 .size = size,
 668         };
 669
 670         if (!mask)
 671                 mask = &rte_flow_item_ipv6_mask;
 672         if (spec) {
 673                 unsigned int i;
 674                 uint32_t vtc_flow_val;
 675                 uint32_t vtc_flow_mask;
 676
 677                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 678                        RTE_DIM(ipv6.val.src_ip));
 679                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 680                        RTE_DIM(ipv6.val.dst_ip));
 681                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 682                        RTE_DIM(ipv6.mask.src_ip));
 683                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 684                        RTE_DIM(ipv6.mask.dst_ip));
 685                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 686                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 687                 ipv6.val.flow_label =
 688                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 689                                          RTE_IPV6_HDR_FL_SHIFT);
 690                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 691                                          RTE_IPV6_HDR_TC_SHIFT;
 692                 ipv6.val.next_hdr = spec->hdr.proto;
 693                 ipv6.mask.flow_label =
 694                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 695                                          RTE_IPV6_HDR_FL_SHIFT);
 696                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 697                                           RTE_IPV6_HDR_TC_SHIFT;
 698                 ipv6.mask.next_hdr = mask->hdr.proto;
 699                 /* Remove unwanted bits from values. */
 700                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 701                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 702                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 703                 }
 704                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 705                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 706                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 707         }
 708         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 709 }
 710
 711 /**
 712  * Convert the @p item into a Verbs specification. This function assumes that
 713  * the input is valid and that there is space to insert the requested item
 714  * into the flow.
 715  *
 716  * @param[in, out] dev_flow
 717  *   Pointer to dev_flow structure.
 718  * @param[in] item
 719  *   Item specification.
 720  * @param[in] item_flags
 721  *   Parsed item flags.
 722  */
 723 static void
 724 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 725                               const struct rte_flow_item *item,
 726                               uint64_t item_flags __rte_unused)
 727 {
 728         const struct rte_flow_item_tcp *spec = item->spec;
 729         const struct rte_flow_item_tcp *mask = item->mask;
 730         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 731         struct ibv_flow_spec_tcp_udp tcp = {
 732                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 733                 .size = size,
 734         };
 735
 736         if (!mask)
 737                 mask = &rte_flow_item_tcp_mask;
 738         if (spec) {
 739                 tcp.val.dst_port = spec->hdr.dst_port;
 740                 tcp.val.src_port = spec->hdr.src_port;
 741                 tcp.mask.dst_port = mask->hdr.dst_port;
 742                 tcp.mask.src_port = mask->hdr.src_port;
 743                 /* Remove unwanted bits from values. */
 744                 tcp.val.src_port &= tcp.mask.src_port;
 745                 tcp.val.dst_port &= tcp.mask.dst_port;
 746         }
 747         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 748 }
 749
 750 /**
 751  * Convert the @p item into a Verbs specification. This function assumes that
 752  * the input is valid and that there is space to insert the requested item
 753  * into the flow.
 754  *
 755  * @param[in, out] dev_flow
 756  *   Pointer to dev_flow structure.
 757  * @param[in] item
 758  *   Item specification.
 759  * @param[in] item_flags
 760  *   Parsed item flags.
 761  */
 762 static void
 763 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 764                               const struct rte_flow_item *item,
 765                               uint64_t item_flags __rte_unused)
 766 {
 767         const struct rte_flow_item_udp *spec = item->spec;
 768         const struct rte_flow_item_udp *mask = item->mask;
 769         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 770         struct ibv_flow_spec_tcp_udp udp = {
 771                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 772                 .size = size,
 773         };
 774
 775         if (!mask)
 776                 mask = &rte_flow_item_udp_mask;
 777         if (spec) {
 778                 udp.val.dst_port = spec->hdr.dst_port;
 779                 udp.val.src_port = spec->hdr.src_port;
 780                 udp.mask.dst_port = mask->hdr.dst_port;
 781                 udp.mask.src_port = mask->hdr.src_port;
 782                 /* Remove unwanted bits from values. */
 783                 udp.val.src_port &= udp.mask.src_port;
 784                 udp.val.dst_port &= udp.mask.dst_port;
 785         }
 786         item++;
 787         while (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 788                 item++;
 789         if (!(udp.val.dst_port & udp.mask.dst_port)) {
 790                 switch ((item)->type) {
 791                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 792                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN);
 793                         udp.mask.dst_port = 0xffff;
 794                         break;
 795                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 796                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE);
 797                         udp.mask.dst_port = 0xffff;
 798                         break;
 799                 case RTE_FLOW_ITEM_TYPE_MPLS:
 800                         udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS);
 801                         udp.mask.dst_port = 0xffff;
 802                         break;
 803                 default:
 804                         break;
 805                 }
 806         }
 807
 808         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 809 }
 810
 811 /**
 812  * Convert the @p item into a Verbs specification. This function assumes that
 813  * the input is valid and that there is space to insert the requested item
 814  * into the flow.
 815  *
 816  * @param[in, out] dev_flow
 817  *   Pointer to dev_flow structure.
 818  * @param[in] item
 819  *   Item specification.
 820  * @param[in] item_flags
 821  *   Parsed item flags.
 822  */
 823 static void
 824 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 825                                 const struct rte_flow_item *item,
 826                                 uint64_t item_flags __rte_unused)
 827 {
 828         const struct rte_flow_item_vxlan *spec = item->spec;
 829         const struct rte_flow_item_vxlan *mask = item->mask;
 830         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 831         struct ibv_flow_spec_tunnel vxlan = {
 832                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 833                 .size = size,
 834         };
 835         union vni {
 836                 uint32_t vlan_id;
 837                 uint8_t vni[4];
 838         } id = { .vlan_id = 0, };
 839
 840         if (!mask)
 841                 mask = &rte_flow_item_vxlan_mask;
 842         if (spec) {
 843                 memcpy(&id.vni[1], spec->vni, 3);
 844                 vxlan.val.tunnel_id = id.vlan_id;
 845                 memcpy(&id.vni[1], mask->vni, 3);
 846                 vxlan.mask.tunnel_id = id.vlan_id;
 847                 /* Remove unwanted bits from values. */
 848                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 849         }
 850         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 851 }
 852
 853 /**
 854  * Convert the @p item into a Verbs specification. This function assumes that
 855  * the input is valid and that there is space to insert the requested item
 856  * into the flow.
 857  *
 858  * @param[in, out] dev_flow
 859  *   Pointer to dev_flow structure.
 860  * @param[in] item
 861  *   Item specification.
 862  * @param[in] item_flags
 863  *   Parsed item flags.
 864  */
 865 static void
 866 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 867                                     const struct rte_flow_item *item,
 868                                     uint64_t item_flags __rte_unused)
 869 {
 870         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 871         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 872         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 873         struct ibv_flow_spec_tunnel vxlan_gpe = {
 874                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 875                 .size = size,
 876         };
 877         union vni {
 878                 uint32_t vlan_id;
 879                 uint8_t vni[4];
 880         } id = { .vlan_id = 0, };
 881
 882         if (!mask)
 883                 mask = &rte_flow_item_vxlan_gpe_mask;
 884         if (spec) {
 885                 memcpy(&id.vni[1], spec->vni, 3);
 886                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 887                 memcpy(&id.vni[1], mask->vni, 3);
 888                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 889                 /* Remove unwanted bits from values. */
 890                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 891         }
 892         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 893 }
 894
 895 /**
 896  * Update the protocol in Verbs IPv4/IPv6 spec.
 897  *
 898  * @param[in, out] attr
 899  *   Pointer to Verbs attributes structure.
 900  * @param[in] search
 901  *   Specification type to search in order to update the IP protocol.
 902  * @param[in] protocol
 903  *   Protocol value to set if none is present in the specification.
 904  */
 905 static void
 906 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 907                                        enum ibv_flow_spec_type search,
 908                                        uint8_t protocol)
 909 {
 910         unsigned int i;
 911         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 912                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 913
 914         if (!attr)
 915                 return;
 916         for (i = 0; i != attr->num_of_specs; ++i) {
 917                 if (hdr->type == search) {
 918                         union {
 919                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 920                                 struct ibv_flow_spec_ipv6 *ipv6;
 921                         } ip;
 922
 923                         switch (search) {
 924                         case IBV_FLOW_SPEC_IPV4_EXT:
 925                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 926                                 if (!ip.ipv4->val.proto) {
 927                                         ip.ipv4->val.proto = protocol;
 928                                         ip.ipv4->mask.proto = 0xff;
 929                                 }
 930                                 break;
 931                         case IBV_FLOW_SPEC_IPV6:
 932                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 933                                 if (!ip.ipv6->val.next_hdr) {
 934                                         ip.ipv6->val.next_hdr = protocol;
 935                                         ip.ipv6->mask.next_hdr = 0xff;
 936                                 }
 937                                 break;
 938                         default:
 939                                 break;
 940                         }
 941                         break;
 942                 }
 943                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 944         }
 945 }
 946
 947 /**
 948  * Convert the @p item into a Verbs specification. This function assumes that
 949  * the input is valid and that there is space to insert the requested item
 950  * into the flow.
 951  *
 952  * @param[in, out] dev_flow
 953  *   Pointer to dev_flow structure.
 954  * @param[in] item
 955  *   Item specification.
 956  * @param[in] item_flags
 957  *   Parsed item flags.
 958  */
 959 static void
 960 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 961                               const struct rte_flow_item *item __rte_unused,
 962                               uint64_t item_flags)
 963 {
 964         struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
 965 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 966         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 967         struct ibv_flow_spec_tunnel tunnel = {
 968                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 969                 .size = size,
 970         };
 971 #else
 972         const struct rte_flow_item_gre *spec = item->spec;
 973         const struct rte_flow_item_gre *mask = item->mask;
 974         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 975         struct ibv_flow_spec_gre tunnel = {
 976                 .type = IBV_FLOW_SPEC_GRE,
 977                 .size = size,
 978         };
 979
 980         if (!mask)
 981                 mask = &rte_flow_item_gre_mask;
 982         if (spec) {
 983                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 984                 tunnel.val.protocol = spec->protocol;
 985                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 986                 tunnel.mask.protocol = mask->protocol;
 987                 /* Remove unwanted bits from values. */
 988                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 989                 tunnel.val.protocol &= tunnel.mask.protocol;
 990                 tunnel.val.key &= tunnel.mask.key;
 991         }
 992 #endif
 993         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 994                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 995                                                        IBV_FLOW_SPEC_IPV4_EXT,
 996                                                        IPPROTO_GRE);
 997         else
 998                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 999                                                        IBV_FLOW_SPEC_IPV6,
1000                                                        IPPROTO_GRE);
1001         flow_verbs_spec_add(verbs, &tunnel, size);
1002 }
1003
1004 /**
1005  * Convert the @p action into a Verbs specification. This function assumes that
1006  * the input is valid and that there is space to insert the requested action
1007  * into the flow. This function also return the action that was added.
1008  *
1009  * @param[in, out] dev_flow
1010  *   Pointer to dev_flow structure.
1011  * @param[in] item
1012  *   Item specification.
1013  * @param[in] item_flags
1014  *   Parsed item flags.
1015  */
1016 static void
1017 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
1018                                const struct rte_flow_item *item __rte_unused,
1019                                uint64_t item_flags __rte_unused)
1020 {
1021 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1022         const struct rte_flow_item_mpls *spec = item->spec;
1023         const struct rte_flow_item_mpls *mask = item->mask;
1024         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
1025         struct ibv_flow_spec_mpls mpls = {
1026                 .type = IBV_FLOW_SPEC_MPLS,
1027                 .size = size,
1028         };
1029
1030         if (!mask)
1031                 mask = &rte_flow_item_mpls_mask;
1032         if (spec) {
1033                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
1034                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
1035                 /* Remove unwanted bits from values.  */
1036                 mpls.val.label &= mpls.mask.label;
1037         }
1038         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
1039 #endif
1040 }
1041
1042 /**
1043  * Convert the @p action into a Verbs specification. This function assumes that
1044  * the input is valid and that there is space to insert the requested action
1045  * into the flow.
1046  *
1047  * @param[in] dev_flow
1048  *   Pointer to mlx5_flow.
1049  * @param[in] action
1050  *   Action configuration.
1051  */
1052 static void
1053 flow_verbs_translate_action_drop
1054         (struct mlx5_flow *dev_flow,
1055          const struct rte_flow_action *action __rte_unused)
1056 {
1057         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1058         struct ibv_flow_spec_action_drop drop = {
1059                         .type = IBV_FLOW_SPEC_ACTION_DROP,
1060                         .size = size,
1061         };
1062
1063         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
1064 }
1065
1066 /**
1067  * Convert the @p action into a Verbs specification. This function assumes that
1068  * the input is valid and that there is space to insert the requested action
1069  * into the flow.
1070  *
1071  * @param[in] rss_desc
1072  *   Pointer to mlx5_flow_rss_desc.
1073  * @param[in] action
1074  *   Action configuration.
1075  */
1076 static void
1077 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc,
1078                                   const struct rte_flow_action *action)
1079 {
1080         const struct rte_flow_action_queue *queue = action->conf;
1081
1082         rss_desc->queue[0] = queue->index;
1083         rss_desc->queue_num = 1;
1084 }
1085
1086 /**
1087  * Convert the @p action into a Verbs specification. This function assumes that
1088  * the input is valid and that there is space to insert the requested action
1089  * into the flow.
1090  *
1091  * @param[in] rss_desc
1092  *   Pointer to mlx5_flow_rss_desc.
1093  * @param[in] action
1094  *   Action configuration.
1095  */
1096 static void
1097 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc,
1098                                 const struct rte_flow_action *action)
1099 {
1100         const struct rte_flow_action_rss *rss = action->conf;
1101         const uint8_t *rss_key;
1102
1103         memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t));
1104         rss_desc->queue_num = rss->queue_num;
1105         /* NULL RSS key indicates default RSS key. */
1106         rss_key = !rss->key ? rss_hash_default_key : rss->key;
1107         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
1108         /*
1109          * rss->level and rss.types should be set in advance when expanding
1110          * items for RSS.
1111          */
1112 }
1113
1114 /**
1115  * Convert the @p action into a Verbs specification. This function assumes that
1116  * the input is valid and that there is space to insert the requested action
1117  * into the flow.
1118  *
1119  * @param[in] dev_flow
1120  *   Pointer to mlx5_flow.
1121  * @param[in] action
1122  *   Action configuration.
1123  */
1124 static void
1125 flow_verbs_translate_action_flag
1126         (struct mlx5_flow *dev_flow,
1127          const struct rte_flow_action *action __rte_unused)
1128 {
1129         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1130         struct ibv_flow_spec_action_tag tag = {
1131                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1132                 .size = size,
1133                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1134         };
1135
1136         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1137 }
1138
1139 /**
1140  * Convert the @p action into a Verbs specification. This function assumes that
1141  * the input is valid and that there is space to insert the requested action
1142  * into the flow.
1143  *
1144  * @param[in] dev_flow
1145  *   Pointer to mlx5_flow.
1146  * @param[in] action
1147  *   Action configuration.
1148  */
1149 static void
1150 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1151                                  const struct rte_flow_action *action)
1152 {
1153         const struct rte_flow_action_mark *mark = action->conf;
1154         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1155         struct ibv_flow_spec_action_tag tag = {
1156                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1157                 .size = size,
1158                 .tag_id = mlx5_flow_mark_set(mark->id),
1159         };
1160
1161         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1162 }
1163
1164 /**
1165  * Convert the @p action into a Verbs specification. This function assumes that
1166  * the input is valid and that there is space to insert the requested action
1167  * into the flow.
1168  *
1169  * @param[in] dev
1170  *   Pointer to the Ethernet device structure.
1171  * @param[in] action
1172  *   Action configuration.
1173  * @param[in] dev_flow
1174  *   Pointer to mlx5_flow.
1175  * @param[out] error
1176  *   Pointer to error structure.
1177  *
1178  * @return
1179  *   0 On success else a negative errno value is returned and rte_errno is set.
1180  */
1181 static int
1182 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1183                                   const struct rte_flow_action *action,
1184                                   struct rte_eth_dev *dev,
1185                                   struct rte_flow_error *error)
1186 {
1187         const struct rte_flow_action_count *count = action->conf;
1188         struct rte_flow *flow = dev_flow->flow;
1189 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1190         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1191         struct mlx5_flow_counter_pool *pool;
1192         struct mlx5_flow_counter *cnt = NULL;
1193         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1194         struct ibv_flow_spec_counter_action counter = {
1195                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1196                 .size = size,
1197         };
1198 #endif
1199
1200         if (!flow->counter) {
1201                 flow->counter = flow_verbs_counter_new(dev, count->shared,
1202                                                        count->id);
1203                 if (!flow->counter)
1204                         return rte_flow_error_set(error, rte_errno,
1205                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1206                                                   action,
1207                                                   "cannot get counter"
1208                                                   " context.");
1209         }
1210 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1211         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1212         counter.counter_set_handle =
1213                 ((struct ibv_counter_set *)cnt->dcs_when_active)->handle;
1214         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1215 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1216         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1217         counter.counters = (struct ibv_counters *)cnt->dcs_when_active;
1218         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1219 #endif
1220         return 0;
1221 }
1222
1223 /**
1224  * Internal validation function. For validating both actions and items.
1225  *
1226  * @param[in] dev
1227  *   Pointer to the Ethernet device structure.
1228  * @param[in] attr
1229  *   Pointer to the flow attributes.
1230  * @param[in] items
1231  *   Pointer to the list of items.
1232  * @param[in] actions
1233  *   Pointer to the list of actions.
1234  * @param[in] external
1235  *   This flow rule is created by request external to PMD.
1236  * @param[in] hairpin
1237  *   Number of hairpin TX actions, 0 means classic flow.
1238  * @param[out] error
1239  *   Pointer to the error structure.
1240  *
1241  * @return
1242  *   0 on success, a negative errno value otherwise and rte_errno is set.
1243  */
1244 static int
1245 flow_verbs_validate(struct rte_eth_dev *dev,
1246                     const struct rte_flow_attr *attr,
1247                     const struct rte_flow_item items[],
1248                     const struct rte_flow_action actions[],
1249                     bool external __rte_unused,
1250                     int hairpin __rte_unused,
1251                     struct rte_flow_error *error)
1252 {
1253         int ret;
1254         uint64_t action_flags = 0;
1255         uint64_t item_flags = 0;
1256         uint64_t last_item = 0;
1257         uint8_t next_protocol = 0xff;
1258         uint16_t ether_type = 0;
1259         bool is_empty_vlan = false;
1260         uint16_t udp_dport = 0;
1261
1262         if (items == NULL)
1263                 return -1;
1264         ret = mlx5_flow_validate_attributes(dev, attr, error);
1265         if (ret < 0)
1266                 return ret;
1267         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1268                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1269                 int ret = 0;
1270
1271                 switch (items->type) {
1272                 case RTE_FLOW_ITEM_TYPE_VOID:
1273                         break;
1274                 case RTE_FLOW_ITEM_TYPE_ETH:
1275                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1276                                                           false, error);
1277                         if (ret < 0)
1278                                 return ret;
1279                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1280                                              MLX5_FLOW_LAYER_OUTER_L2;
1281                         if (items->mask != NULL && items->spec != NULL) {
1282                                 ether_type =
1283                                         ((const struct rte_flow_item_eth *)
1284                                          items->spec)->type;
1285                                 ether_type &=
1286                                         ((const struct rte_flow_item_eth *)
1287                                          items->mask)->type;
1288                                 if (ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN))
1289                                         is_empty_vlan = true;
1290                                 ether_type = rte_be_to_cpu_16(ether_type);
1291                         } else {
1292                                 ether_type = 0;
1293                         }
1294                         break;
1295                 case RTE_FLOW_ITEM_TYPE_VLAN:
1296                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1297                                                            dev, error);
1298                         if (ret < 0)
1299                                 return ret;
1300                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1301                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1302                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1303                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1304                         if (items->mask != NULL && items->spec != NULL) {
1305                                 ether_type =
1306                                         ((const struct rte_flow_item_vlan *)
1307                                          items->spec)->inner_type;
1308                                 ether_type &=
1309                                         ((const struct rte_flow_item_vlan *)
1310                                          items->mask)->inner_type;
1311                                 ether_type = rte_be_to_cpu_16(ether_type);
1312                         } else {
1313                                 ether_type = 0;
1314                         }
1315                         is_empty_vlan = false;
1316                         break;
1317                 case RTE_FLOW_ITEM_TYPE_IPV4:
1318                         ret = mlx5_flow_validate_item_ipv4
1319                                                 (items, item_flags,
1320                                                  last_item, ether_type, NULL,
1321                                                  MLX5_ITEM_RANGE_NOT_ACCEPTED,
1322                                                  error);
1323                         if (ret < 0)
1324                                 return ret;
1325                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1326                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1327                         if (items->mask != NULL &&
1328                             ((const struct rte_flow_item_ipv4 *)
1329                              items->mask)->hdr.next_proto_id) {
1330                                 next_protocol =
1331                                         ((const struct rte_flow_item_ipv4 *)
1332                                          (items->spec))->hdr.next_proto_id;
1333                                 next_protocol &=
1334                                         ((const struct rte_flow_item_ipv4 *)
1335                                          (items->mask))->hdr.next_proto_id;
1336                         } else {
1337                                 /* Reset for inner layer. */
1338                                 next_protocol = 0xff;
1339                         }
1340                         break;
1341                 case RTE_FLOW_ITEM_TYPE_IPV6:
1342                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1343                                                            last_item,
1344                                                            ether_type, NULL,
1345                                                            error);
1346                         if (ret < 0)
1347                                 return ret;
1348                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1349                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1350                         if (items->mask != NULL &&
1351                             ((const struct rte_flow_item_ipv6 *)
1352                              items->mask)->hdr.proto) {
1353                                 next_protocol =
1354                                         ((const struct rte_flow_item_ipv6 *)
1355                                          items->spec)->hdr.proto;
1356                                 next_protocol &=
1357                                         ((const struct rte_flow_item_ipv6 *)
1358                                          items->mask)->hdr.proto;
1359                         } else {
1360                                 /* Reset for inner layer. */
1361                                 next_protocol = 0xff;
1362                         }
1363                         break;
1364                 case RTE_FLOW_ITEM_TYPE_UDP:
1365                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1366                                                           next_protocol,
1367                                                           error);
1368                         const struct rte_flow_item_udp *spec = items->spec;
1369                         const struct rte_flow_item_udp *mask = items->mask;
1370                         if (!mask)
1371                                 mask = &rte_flow_item_udp_mask;
1372                         if (spec != NULL)
1373                                 udp_dport = rte_be_to_cpu_16
1374                                                 (spec->hdr.dst_port &
1375                                                  mask->hdr.dst_port);
1376
1377                         if (ret < 0)
1378                                 return ret;
1379                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1380                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1381                         break;
1382                 case RTE_FLOW_ITEM_TYPE_TCP:
1383                         ret = mlx5_flow_validate_item_tcp
1384                                                 (items, item_flags,
1385                                                  next_protocol,
1386                                                  &rte_flow_item_tcp_mask,
1387                                                  error);
1388                         if (ret < 0)
1389                                 return ret;
1390                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1391                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1392                         break;
1393                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1394                         ret = mlx5_flow_validate_item_vxlan(dev, udp_dport,
1395                                                             items, item_flags,
1396                                                             attr, error);
1397                         if (ret < 0)
1398                                 return ret;
1399                         last_item = MLX5_FLOW_LAYER_VXLAN;
1400                         break;
1401                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1402                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1403                                                                 item_flags,
1404                                                                 dev, error);
1405                         if (ret < 0)
1406                                 return ret;
1407                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1408                         break;
1409                 case RTE_FLOW_ITEM_TYPE_GRE:
1410                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1411                                                           next_protocol, error);
1412                         if (ret < 0)
1413                                 return ret;
1414                         last_item = MLX5_FLOW_LAYER_GRE;
1415                         break;
1416                 case RTE_FLOW_ITEM_TYPE_MPLS:
1417                         ret = mlx5_flow_validate_item_mpls(dev, items,
1418                                                            item_flags,
1419                                                            last_item, error);
1420                         if (ret < 0)
1421                                 return ret;
1422                         last_item = MLX5_FLOW_LAYER_MPLS;
1423                         break;
1424                 case RTE_FLOW_ITEM_TYPE_ICMP:
1425                 case RTE_FLOW_ITEM_TYPE_ICMP6:
1426                         return rte_flow_error_set(error, ENOTSUP,
1427                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1428                                                   NULL, "ICMP/ICMP6 "
1429                                                   "item not supported");
1430                 default:
1431                         return rte_flow_error_set(error, ENOTSUP,
1432                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1433                                                   NULL, "item not supported");
1434                 }
1435                 item_flags |= last_item;
1436         }
1437         if (is_empty_vlan)
1438                 return rte_flow_error_set(error, ENOTSUP,
1439                                                  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1440                     "VLAN matching without vid specification is not supported");
1441         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1442                 switch (actions->type) {
1443                 case RTE_FLOW_ACTION_TYPE_VOID:
1444                         break;
1445                 case RTE_FLOW_ACTION_TYPE_FLAG:
1446                         ret = mlx5_flow_validate_action_flag(action_flags,
1447                                                              attr,
1448                                                              error);
1449                         if (ret < 0)
1450                                 return ret;
1451                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1452                         break;
1453                 case RTE_FLOW_ACTION_TYPE_MARK:
1454                         ret = mlx5_flow_validate_action_mark(actions,
1455                                                              action_flags,
1456                                                              attr,
1457                                                              error);
1458                         if (ret < 0)
1459                                 return ret;
1460                         action_flags |= MLX5_FLOW_ACTION_MARK;
1461                         break;
1462                 case RTE_FLOW_ACTION_TYPE_DROP:
1463                         ret = mlx5_flow_validate_action_drop(action_flags,
1464                                                              attr,
1465                                                              error);
1466                         if (ret < 0)
1467                                 return ret;
1468                         action_flags |= MLX5_FLOW_ACTION_DROP;
1469                         break;
1470                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1471                         ret = mlx5_flow_validate_action_queue(actions,
1472                                                               action_flags, dev,
1473                                                               attr,
1474                                                               error);
1475                         if (ret < 0)
1476                                 return ret;
1477                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1478                         break;
1479                 case RTE_FLOW_ACTION_TYPE_RSS:
1480                         ret = mlx5_flow_validate_action_rss(actions,
1481                                                             action_flags, dev,
1482                                                             attr, item_flags,
1483                                                             error);
1484                         if (ret < 0)
1485                                 return ret;
1486                         action_flags |= MLX5_FLOW_ACTION_RSS;
1487                         break;
1488                 case RTE_FLOW_ACTION_TYPE_COUNT:
1489                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1490                         if (ret < 0)
1491                                 return ret;
1492                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1493                         break;
1494                 default:
1495                         return rte_flow_error_set(error, ENOTSUP,
1496                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1497                                                   actions,
1498                                                   "action not supported");
1499                 }
1500         }
1501         /*
1502          * Validate the drop action mutual exclusion with other actions.
1503          * Drop action is mutually-exclusive with any other action, except for
1504          * Count action.
1505          */
1506         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1507             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1508                 return rte_flow_error_set(error, EINVAL,
1509                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1510                                           "Drop action is mutually-exclusive "
1511                                           "with any other action, except for "
1512                                           "Count action");
1513         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1514                 return rte_flow_error_set(error, EINVAL,
1515                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1516                                           "no fate action is found");
1517         return 0;
1518 }
1519
1520 /**
1521  * Calculate the required bytes that are needed for the action part of the verbs
1522  * flow.
1523  *
1524  * @param[in] actions
1525  *   Pointer to the list of actions.
1526  *
1527  * @return
1528  *   The size of the memory needed for all actions.
1529  */
1530 static int
1531 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1532 {
1533         int size = 0;
1534
1535         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1536                 switch (actions->type) {
1537                 case RTE_FLOW_ACTION_TYPE_VOID:
1538                         break;
1539                 case RTE_FLOW_ACTION_TYPE_FLAG:
1540                         size += sizeof(struct ibv_flow_spec_action_tag);
1541                         break;
1542                 case RTE_FLOW_ACTION_TYPE_MARK:
1543                         size += sizeof(struct ibv_flow_spec_action_tag);
1544                         break;
1545                 case RTE_FLOW_ACTION_TYPE_DROP:
1546                         size += sizeof(struct ibv_flow_spec_action_drop);
1547                         break;
1548                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1549                         break;
1550                 case RTE_FLOW_ACTION_TYPE_RSS:
1551                         break;
1552                 case RTE_FLOW_ACTION_TYPE_COUNT:
1553 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1554         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1555                         size += sizeof(struct ibv_flow_spec_counter_action);
1556 #endif
1557                         break;
1558                 default:
1559                         break;
1560                 }
1561         }
1562         return size;
1563 }
1564
1565 /**
1566  * Calculate the required bytes that are needed for the item part of the verbs
1567  * flow.
1568  *
1569  * @param[in] items
1570  *   Pointer to the list of items.
1571  *
1572  * @return
1573  *   The size of the memory needed for all items.
1574  */
1575 static int
1576 flow_verbs_get_items_size(const struct rte_flow_item items[])
1577 {
1578         int size = 0;
1579
1580         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1581                 switch (items->type) {
1582                 case RTE_FLOW_ITEM_TYPE_VOID:
1583                         break;
1584                 case RTE_FLOW_ITEM_TYPE_ETH:
1585                         size += sizeof(struct ibv_flow_spec_eth);
1586                         break;
1587                 case RTE_FLOW_ITEM_TYPE_VLAN:
1588                         size += sizeof(struct ibv_flow_spec_eth);
1589                         break;
1590                 case RTE_FLOW_ITEM_TYPE_IPV4:
1591                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1592                         break;
1593                 case RTE_FLOW_ITEM_TYPE_IPV6:
1594                         size += sizeof(struct ibv_flow_spec_ipv6);
1595                         break;
1596                 case RTE_FLOW_ITEM_TYPE_UDP:
1597                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1598                         break;
1599                 case RTE_FLOW_ITEM_TYPE_TCP:
1600                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1601                         break;
1602                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1603                         size += sizeof(struct ibv_flow_spec_tunnel);
1604                         break;
1605                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1606                         size += sizeof(struct ibv_flow_spec_tunnel);
1607                         break;
1608 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1609                 case RTE_FLOW_ITEM_TYPE_GRE:
1610                         size += sizeof(struct ibv_flow_spec_gre);
1611                         break;
1612                 case RTE_FLOW_ITEM_TYPE_MPLS:
1613                         size += sizeof(struct ibv_flow_spec_mpls);
1614                         break;
1615 #else
1616                 case RTE_FLOW_ITEM_TYPE_GRE:
1617                         size += sizeof(struct ibv_flow_spec_tunnel);
1618                         break;
1619 #endif
1620                 default:
1621                         break;
1622                 }
1623         }
1624         return size;
1625 }
1626
1627 /**
1628  * Internal preparation function. Allocate mlx5_flow with the required size.
1629  * The required size is calculate based on the actions and items. This function
1630  * also returns the detected actions and items for later use.
1631  *
1632  * @param[in] dev
1633  *   Pointer to Ethernet device.
1634  * @param[in] attr
1635  *   Pointer to the flow attributes.
1636  * @param[in] items
1637  *   Pointer to the list of items.
1638  * @param[in] actions
1639  *   Pointer to the list of actions.
1640  * @param[out] error
1641  *   Pointer to the error structure.
1642  *
1643  * @return
1644  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1645  *   is set.
1646  */
1647 static struct mlx5_flow *
1648 flow_verbs_prepare(struct rte_eth_dev *dev,
1649                    const struct rte_flow_attr *attr __rte_unused,
1650                    const struct rte_flow_item items[],
1651                    const struct rte_flow_action actions[],
1652                    struct rte_flow_error *error)
1653 {
1654         size_t size = 0;
1655         uint32_t handle_idx = 0;
1656         struct mlx5_flow *dev_flow;
1657         struct mlx5_flow_handle *dev_handle;
1658         struct mlx5_priv *priv = dev->data->dev_private;
1659         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1660
1661         MLX5_ASSERT(wks);
1662         size += flow_verbs_get_actions_size(actions);
1663         size += flow_verbs_get_items_size(items);
1664         if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1665                 rte_flow_error_set(error, E2BIG,
1666                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1667                                    "Verbs spec/action size too large");
1668                 return NULL;
1669         }
1670         /* In case of corrupting the memory. */
1671         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1672                 rte_flow_error_set(error, ENOSPC,
1673                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1674                                    "not free temporary device flow");
1675                 return NULL;
1676         }
1677         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1678                                    &handle_idx);
1679         if (!dev_handle) {
1680                 rte_flow_error_set(error, ENOMEM,
1681                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1682                                    "not enough memory to create flow handle");
1683                 return NULL;
1684         }
1685         MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows));
1686         dev_flow = &wks->flows[wks->flow_idx++];
1687         dev_flow->handle = dev_handle;
1688         dev_flow->handle_idx = handle_idx;
1689         /* Memcpy is used, only size needs to be cleared to 0. */
1690         dev_flow->verbs.size = 0;
1691         dev_flow->verbs.attr.num_of_specs = 0;
1692         dev_flow->ingress = attr->ingress;
1693         dev_flow->hash_fields = 0;
1694         /* Need to set transfer attribute: not supported in Verbs mode. */
1695         return dev_flow;
1696 }
1697
1698 /**
1699  * Fill the flow with verb spec.
1700  *
1701  * @param[in] dev
1702  *   Pointer to Ethernet device.
1703  * @param[in, out] dev_flow
1704  *   Pointer to the mlx5 flow.
1705  * @param[in] attr
1706  *   Pointer to the flow attributes.
1707  * @param[in] items
1708  *   Pointer to the list of items.
1709  * @param[in] actions
1710  *   Pointer to the list of actions.
1711  * @param[out] error
1712  *   Pointer to the error structure.
1713  *
1714  * @return
1715  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1716  */
1717 static int
1718 flow_verbs_translate(struct rte_eth_dev *dev,
1719                      struct mlx5_flow *dev_flow,
1720                      const struct rte_flow_attr *attr,
1721                      const struct rte_flow_item items[],
1722                      const struct rte_flow_action actions[],
1723                      struct rte_flow_error *error)
1724 {
1725         uint64_t item_flags = 0;
1726         uint64_t action_flags = 0;
1727         uint64_t priority = attr->priority;
1728         uint32_t subpriority = 0;
1729         struct mlx5_priv *priv = dev->data->dev_private;
1730         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1731         struct mlx5_flow_rss_desc *rss_desc;
1732
1733         MLX5_ASSERT(wks);
1734         rss_desc = &wks->rss_desc;
1735         if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1736                 priority = priv->config.flow_prio - 1;
1737         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1738                 int ret;
1739
1740                 switch (actions->type) {
1741                 case RTE_FLOW_ACTION_TYPE_VOID:
1742                         break;
1743                 case RTE_FLOW_ACTION_TYPE_FLAG:
1744                         flow_verbs_translate_action_flag(dev_flow, actions);
1745                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1746                         dev_flow->handle->mark = 1;
1747                         break;
1748                 case RTE_FLOW_ACTION_TYPE_MARK:
1749                         flow_verbs_translate_action_mark(dev_flow, actions);
1750                         action_flags |= MLX5_FLOW_ACTION_MARK;
1751                         dev_flow->handle->mark = 1;
1752                         break;
1753                 case RTE_FLOW_ACTION_TYPE_DROP:
1754                         flow_verbs_translate_action_drop(dev_flow, actions);
1755                         action_flags |= MLX5_FLOW_ACTION_DROP;
1756                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
1757                         break;
1758                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1759                         flow_verbs_translate_action_queue(rss_desc, actions);
1760                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1761                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1762                         break;
1763                 case RTE_FLOW_ACTION_TYPE_RSS:
1764                         flow_verbs_translate_action_rss(rss_desc, actions);
1765                         action_flags |= MLX5_FLOW_ACTION_RSS;
1766                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1767                         break;
1768                 case RTE_FLOW_ACTION_TYPE_COUNT:
1769                         ret = flow_verbs_translate_action_count(dev_flow,
1770                                                                 actions,
1771                                                                 dev, error);
1772                         if (ret < 0)
1773                                 return ret;
1774                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1775                         break;
1776                 default:
1777                         return rte_flow_error_set(error, ENOTSUP,
1778                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1779                                                   actions,
1780                                                   "action not supported");
1781                 }
1782         }
1783         dev_flow->act_flags = action_flags;
1784         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1785                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1786
1787                 switch (items->type) {
1788                 case RTE_FLOW_ITEM_TYPE_VOID:
1789                         break;
1790                 case RTE_FLOW_ITEM_TYPE_ETH:
1791                         flow_verbs_translate_item_eth(dev_flow, items,
1792                                                       item_flags);
1793                         subpriority = MLX5_PRIORITY_MAP_L2;
1794                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1795                                                MLX5_FLOW_LAYER_OUTER_L2;
1796                         break;
1797                 case RTE_FLOW_ITEM_TYPE_VLAN:
1798                         flow_verbs_translate_item_vlan(dev_flow, items,
1799                                                        item_flags);
1800                         subpriority = MLX5_PRIORITY_MAP_L2;
1801                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1802                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1803                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1804                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1805                         break;
1806                 case RTE_FLOW_ITEM_TYPE_IPV4:
1807                         flow_verbs_translate_item_ipv4(dev_flow, items,
1808                                                        item_flags);
1809                         subpriority = MLX5_PRIORITY_MAP_L3;
1810                         dev_flow->hash_fields |=
1811                                 mlx5_flow_hashfields_adjust
1812                                         (rss_desc, tunnel,
1813                                          MLX5_IPV4_LAYER_TYPES,
1814                                          MLX5_IPV4_IBV_RX_HASH);
1815                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1816                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1817                         break;
1818                 case RTE_FLOW_ITEM_TYPE_IPV6:
1819                         flow_verbs_translate_item_ipv6(dev_flow, items,
1820                                                        item_flags);
1821                         subpriority = MLX5_PRIORITY_MAP_L3;
1822                         dev_flow->hash_fields |=
1823                                 mlx5_flow_hashfields_adjust
1824                                         (rss_desc, tunnel,
1825                                          MLX5_IPV6_LAYER_TYPES,
1826                                          MLX5_IPV6_IBV_RX_HASH);
1827                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1828                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1829                         break;
1830                 case RTE_FLOW_ITEM_TYPE_TCP:
1831                         flow_verbs_translate_item_tcp(dev_flow, items,
1832                                                       item_flags);
1833                         subpriority = MLX5_PRIORITY_MAP_L4;
1834                         if (dev_flow->hash_fields != 0)
1835                                 dev_flow->hash_fields |=
1836                                         mlx5_flow_hashfields_adjust
1837                                         (rss_desc, tunnel, ETH_RSS_TCP,
1838                                          (IBV_RX_HASH_SRC_PORT_TCP |
1839                                           IBV_RX_HASH_DST_PORT_TCP));
1840                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1841                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1842                         break;
1843                 case RTE_FLOW_ITEM_TYPE_UDP:
1844                         flow_verbs_translate_item_udp(dev_flow, items,
1845                                                       item_flags);
1846                         subpriority = MLX5_PRIORITY_MAP_L4;
1847                         if (dev_flow->hash_fields != 0)
1848                                 dev_flow->hash_fields |=
1849                                         mlx5_flow_hashfields_adjust
1850                                         (rss_desc, tunnel, ETH_RSS_UDP,
1851                                          (IBV_RX_HASH_SRC_PORT_UDP |
1852                                           IBV_RX_HASH_DST_PORT_UDP));
1853                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1854                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1855                         break;
1856                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1857                         flow_verbs_translate_item_vxlan(dev_flow, items,
1858                                                         item_flags);
1859                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1860                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1861                         break;
1862                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1863                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1864                                                             item_flags);
1865                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1866                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1867                         break;
1868                 case RTE_FLOW_ITEM_TYPE_GRE:
1869                         flow_verbs_translate_item_gre(dev_flow, items,
1870                                                       item_flags);
1871                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1872                         item_flags |= MLX5_FLOW_LAYER_GRE;
1873                         break;
1874                 case RTE_FLOW_ITEM_TYPE_MPLS:
1875                         flow_verbs_translate_item_mpls(dev_flow, items,
1876                                                        item_flags);
1877                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1878                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1879                         break;
1880                 default:
1881                         return rte_flow_error_set(error, ENOTSUP,
1882                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1883                                                   NULL, "item not supported");
1884                 }
1885         }
1886         dev_flow->handle->layers = item_flags;
1887         /* Other members of attr will be ignored. */
1888         dev_flow->verbs.attr.priority =
1889                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1890         dev_flow->verbs.attr.port = (uint8_t)priv->dev_port;
1891         return 0;
1892 }
1893
1894 /**
1895  * Remove the flow from the NIC but keeps it in memory.
1896  *
1897  * @param[in] dev
1898  *   Pointer to the Ethernet device structure.
1899  * @param[in, out] flow
1900  *   Pointer to flow structure.
1901  */
1902 static void
1903 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1904 {
1905         struct mlx5_priv *priv = dev->data->dev_private;
1906         struct mlx5_flow_handle *handle;
1907         uint32_t handle_idx;
1908
1909         if (!flow)
1910                 return;
1911         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1912                        handle_idx, handle, next) {
1913                 if (handle->drv_flow) {
1914                         claim_zero(mlx5_glue->destroy_flow(handle->drv_flow));
1915                         handle->drv_flow = NULL;
1916                 }
1917                 /* hrxq is union, don't touch it only the flag is set. */
1918                 if (handle->rix_hrxq &&
1919                     handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1920                         mlx5_hrxq_release(dev, handle->rix_hrxq);
1921                         handle->rix_hrxq = 0;
1922                 }
1923                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1924                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1925         }
1926 }
1927
1928 /**
1929  * Remove the flow from the NIC and the memory.
1930  *
1931  * @param[in] dev
1932  *   Pointer to the Ethernet device structure.
1933  * @param[in, out] flow
1934  *   Pointer to flow structure.
1935  */
1936 static void
1937 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1938 {
1939         struct mlx5_priv *priv = dev->data->dev_private;
1940         struct mlx5_flow_handle *handle;
1941
1942         if (!flow)
1943                 return;
1944         flow_verbs_remove(dev, flow);
1945         while (flow->dev_handles) {
1946                 uint32_t tmp_idx = flow->dev_handles;
1947
1948                 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1949                                    tmp_idx);
1950                 if (!handle)
1951                         return;
1952                 flow->dev_handles = handle->next.next;
1953                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1954                            tmp_idx);
1955         }
1956         if (flow->counter) {
1957                 flow_verbs_counter_release(dev, flow->counter);
1958                 flow->counter = 0;
1959         }
1960 }
1961
1962 /**
1963  * Apply the flow to the NIC.
1964  *
1965  * @param[in] dev
1966  *   Pointer to the Ethernet device structure.
1967  * @param[in, out] flow
1968  *   Pointer to flow structure.
1969  * @param[out] error
1970  *   Pointer to error structure.
1971  *
1972  * @return
1973  *   0 on success, a negative errno value otherwise and rte_errno is set.
1974  */
1975 static int
1976 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1977                  struct rte_flow_error *error)
1978 {
1979         struct mlx5_priv *priv = dev->data->dev_private;
1980         struct mlx5_flow_handle *handle;
1981         struct mlx5_flow *dev_flow;
1982         struct mlx5_hrxq *hrxq;
1983         uint32_t dev_handles;
1984         int err;
1985         int idx;
1986         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1987
1988         MLX5_ASSERT(wks);
1989         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
1990                 dev_flow = &wks->flows[idx];
1991                 handle = dev_flow->handle;
1992                 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1993                         MLX5_ASSERT(priv->drop_queue.hrxq);
1994                         hrxq = priv->drop_queue.hrxq;
1995                 } else {
1996                         uint32_t hrxq_idx;
1997                         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
1998
1999                         MLX5_ASSERT(rss_desc->queue_num);
2000                         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
2001                         rss_desc->hash_fields = dev_flow->hash_fields;
2002                         rss_desc->tunnel = !!(handle->layers &
2003                                               MLX5_FLOW_LAYER_TUNNEL);
2004                         rss_desc->shared_rss = 0;
2005                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
2006                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
2007                                               hrxq_idx);
2008                         if (!hrxq) {
2009                                 rte_flow_error_set
2010                                         (error, rte_errno,
2011                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2012                                          "cannot get hash queue");
2013                                 goto error;
2014                         }
2015                         handle->rix_hrxq = hrxq_idx;
2016                 }
2017                 MLX5_ASSERT(hrxq);
2018                 handle->drv_flow = mlx5_glue->create_flow
2019                                         (hrxq->qp, &dev_flow->verbs.attr);
2020                 if (!handle->drv_flow) {
2021                         rte_flow_error_set(error, errno,
2022                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2023                                            NULL,
2024                                            "hardware refuses to create flow");
2025                         goto error;
2026                 }
2027                 if (priv->vmwa_context &&
2028                     handle->vf_vlan.tag && !handle->vf_vlan.created) {
2029                         /*
2030                          * The rule contains the VLAN pattern.
2031                          * For VF we are going to create VLAN
2032                          * interface to make hypervisor set correct
2033                          * e-Switch vport context.
2034                          */
2035                         mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
2036                 }
2037         }
2038         return 0;
2039 error:
2040         err = rte_errno; /* Save rte_errno before cleanup. */
2041         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2042                        dev_handles, handle, next) {
2043                 /* hrxq is union, don't touch it only the flag is set. */
2044                 if (handle->rix_hrxq &&
2045                     handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
2046                         mlx5_hrxq_release(dev, handle->rix_hrxq);
2047                         handle->rix_hrxq = 0;
2048                 }
2049                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
2050                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
2051         }
2052         rte_errno = err; /* Restore rte_errno. */
2053         return -rte_errno;
2054 }
2055
2056 /**
2057  * Query a flow.
2058  *
2059  * @see rte_flow_query()
2060  * @see rte_flow_ops
2061  */
2062 static int
2063 flow_verbs_query(struct rte_eth_dev *dev,
2064                  struct rte_flow *flow,
2065                  const struct rte_flow_action *actions,
2066                  void *data,
2067                  struct rte_flow_error *error)
2068 {
2069         int ret = -EINVAL;
2070
2071         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2072                 switch (actions->type) {
2073                 case RTE_FLOW_ACTION_TYPE_VOID:
2074                         break;
2075                 case RTE_FLOW_ACTION_TYPE_COUNT:
2076                         ret = flow_verbs_counter_query(dev, flow, data, error);
2077                         break;
2078                 default:
2079                         return rte_flow_error_set(error, ENOTSUP,
2080                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2081                                                   actions,
2082                                                   "action not supported");
2083                 }
2084         }
2085         return ret;
2086 }
2087
2088 static int
2089 flow_verbs_sync_domain(struct rte_eth_dev *dev, uint32_t domains,
2090                        uint32_t flags)
2091 {
2092         RTE_SET_USED(dev);
2093         RTE_SET_USED(domains);
2094         RTE_SET_USED(flags);
2095
2096         return 0;
2097 }
2098
2099 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
2100         .validate = flow_verbs_validate,
2101         .prepare = flow_verbs_prepare,
2102         .translate = flow_verbs_translate,
2103         .apply = flow_verbs_apply,
2104         .remove = flow_verbs_remove,
2105         .destroy = flow_verbs_destroy,
2106         .query = flow_verbs_query,
2107         .sync_domain = flow_verbs_sync_domain,
2108 };