drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_ip.h>
  18
  19 #include <mlx5_glue.h>
  20 #include <mlx5_prm.h>
  21 #include <mlx5_malloc.h>
  22
  23 #include "mlx5_defs.h"
  24 #include "mlx5.h"
  25 #include "mlx5_flow.h"
  26 #include "mlx5_rx.h"
  27
  28 #define VERBS_SPEC_INNER(item_flags) \
  29         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  30
  31 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
  32 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
  33         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
  34 };
  35
  36 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
  37 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
  38         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
  39         { 9, 10, 11 }, { 12, 13, 14 },
  40 };
  41
  42 /* Verbs specification header. */
  43 struct ibv_spec_header {
  44         enum ibv_flow_spec_type type;
  45         uint16_t size;
  46 };
  47
  48 /**
  49  * Discover the maximum number of priority available.
  50  *
  51  * @param[in] dev
  52  *   Pointer to the Ethernet device structure.
  53  *
  54  * @return
  55  *   number of supported flow priority on success, a negative errno
  56  *   value otherwise and rte_errno is set.
  57  */
  58 int
  59 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
  60 {
  61         struct mlx5_priv *priv = dev->data->dev_private;
  62         struct {
  63                 struct ibv_flow_attr attr;
  64                 struct ibv_flow_spec_eth eth;
  65                 struct ibv_flow_spec_action_drop drop;
  66         } flow_attr = {
  67                 .attr = {
  68                         .num_of_specs = 2,
  69                         .port = (uint8_t)priv->dev_port,
  70                 },
  71                 .eth = {
  72                         .type = IBV_FLOW_SPEC_ETH,
  73                         .size = sizeof(struct ibv_flow_spec_eth),
  74                 },
  75                 .drop = {
  76                         .size = sizeof(struct ibv_flow_spec_action_drop),
  77                         .type = IBV_FLOW_SPEC_ACTION_DROP,
  78                 },
  79         };
  80         struct ibv_flow *flow;
  81         struct mlx5_hrxq *drop = priv->drop_queue.hrxq;
  82         uint16_t vprio[] = { 8, 16 };
  83         int i;
  84         int priority = 0;
  85
  86         if (!drop->qp) {
  87                 rte_errno = ENOTSUP;
  88                 return -rte_errno;
  89         }
  90         for (i = 0; i != RTE_DIM(vprio); i++) {
  91                 flow_attr.attr.priority = vprio[i] - 1;
  92                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
  93                 if (!flow)
  94                         break;
  95                 claim_zero(mlx5_glue->destroy_flow(flow));
  96                 priority = vprio[i];
  97         }
  98         switch (priority) {
  99         case 8:
 100                 priority = RTE_DIM(priority_map_3);
 101                 break;
 102         case 16:
 103                 priority = RTE_DIM(priority_map_5);
 104                 break;
 105         default:
 106                 rte_errno = ENOTSUP;
 107                 DRV_LOG(ERR,
 108                         "port %u verbs maximum priority: %d expected 8/16",
 109                         dev->data->port_id, priority);
 110                 return -rte_errno;
 111         }
 112         DRV_LOG(INFO, "port %u supported flow priorities:"
 113                 " 0-%d for ingress or egress root table,"
 114                 " 0-%d for non-root table or transfer root table.",
 115                 dev->data->port_id, priority - 2,
 116                 MLX5_NON_ROOT_FLOW_MAX_PRIO - 1);
 117         return priority;
 118 }
 119
 120 /**
 121  * Adjust flow priority based on the highest layer and the request priority.
 122  *
 123  * @param[in] dev
 124  *   Pointer to the Ethernet device structure.
 125  * @param[in] priority
 126  *   The rule base priority.
 127  * @param[in] subpriority
 128  *   The priority based on the items.
 129  *
 130  * @return
 131  *   The new priority.
 132  */
 133 uint32_t
 134 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 135                                    uint32_t subpriority)
 136 {
 137         uint32_t res = 0;
 138         struct mlx5_priv *priv = dev->data->dev_private;
 139
 140         switch (priv->config.flow_prio) {
 141         case RTE_DIM(priority_map_3):
 142                 res = priority_map_3[priority][subpriority];
 143                 break;
 144         case RTE_DIM(priority_map_5):
 145                 res = priority_map_5[priority][subpriority];
 146                 break;
 147         }
 148         return  res;
 149 }
 150
 151 /**
 152  * Get Verbs flow counter by index.
 153  *
 154  * @param[in] dev
 155  *   Pointer to the Ethernet device structure.
 156  * @param[in] idx
 157  *   mlx5 flow counter index in the container.
 158  * @param[out] ppool
 159  *   mlx5 flow counter pool in the container,
 160  *
 161  * @return
 162  *   A pointer to the counter, NULL otherwise.
 163  */
 164 static struct mlx5_flow_counter *
 165 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
 166                               uint32_t idx,
 167                               struct mlx5_flow_counter_pool **ppool)
 168 {
 169         struct mlx5_priv *priv = dev->data->dev_private;
 170         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 171         struct mlx5_flow_counter_pool *pool;
 172
 173         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
 174         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
 175         MLX5_ASSERT(pool);
 176         if (ppool)
 177                 *ppool = pool;
 178         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
 179 }
 180
 181 /**
 182  * Create Verbs flow counter with Verbs library.
 183  *
 184  * @param[in] dev
 185  *   Pointer to the Ethernet device structure.
 186  * @param[in, out] counter
 187  *   mlx5 flow counter object, contains the counter id,
 188  *   handle of created Verbs flow counter is returned
 189  *   in cs field (if counters are supported).
 190  *
 191  * @return
 192  *   0 On success else a negative errno value is returned
 193  *   and rte_errno is set.
 194  */
 195 static int
 196 flow_verbs_counter_create(struct rte_eth_dev *dev,
 197                           struct mlx5_flow_counter *counter)
 198 {
 199 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 200         struct mlx5_priv *priv = dev->data->dev_private;
 201         struct ibv_context *ctx = priv->sh->ctx;
 202         struct ibv_counter_set_init_attr init = {
 203                          .counter_set_id = counter->shared_info.id};
 204
 205         counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init);
 206         if (!counter->dcs_when_free) {
 207                 rte_errno = ENOTSUP;
 208                 return -ENOTSUP;
 209         }
 210         return 0;
 211 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 212         struct mlx5_priv *priv = dev->data->dev_private;
 213         struct ibv_context *ctx = priv->sh->ctx;
 214         struct ibv_counters_init_attr init = {0};
 215         struct ibv_counter_attach_attr attach;
 216         int ret;
 217
 218         memset(&attach, 0, sizeof(attach));
 219         counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init);
 220         if (!counter->dcs_when_free) {
 221                 rte_errno = ENOTSUP;
 222                 return -ENOTSUP;
 223         }
 224         attach.counter_desc = IBV_COUNTER_PACKETS;
 225         attach.index = 0;
 226         ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL);
 227         if (!ret) {
 228                 attach.counter_desc = IBV_COUNTER_BYTES;
 229                 attach.index = 1;
 230                 ret = mlx5_glue->attach_counters
 231                                         (counter->dcs_when_free, &attach, NULL);
 232         }
 233         if (ret) {
 234                 claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free));
 235                 counter->dcs_when_free = NULL;
 236                 rte_errno = ret;
 237                 return -ret;
 238         }
 239         return 0;
 240 #else
 241         (void)dev;
 242         (void)counter;
 243         rte_errno = ENOTSUP;
 244         return -ENOTSUP;
 245 #endif
 246 }
 247
 248 /**
 249  * Get a flow counter.
 250  *
 251  * @param[in] dev
 252  *   Pointer to the Ethernet device structure.
 253  * @param[in] shared
 254  *   Indicate if this counter is shared with other flows.
 255  * @param[in] id
 256  *   Counter identifier.
 257  *
 258  * @return
 259  *   Index to the counter, 0 otherwise and rte_errno is set.
 260  */
 261 static uint32_t
 262 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 263 {
 264         struct mlx5_priv *priv = dev->data->dev_private;
 265         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 266         struct mlx5_flow_counter_pool *pool = NULL;
 267         struct mlx5_flow_counter *cnt = NULL;
 268         union mlx5_l3t_data data;
 269         uint32_t n_valid = cmng->n_valid;
 270         uint32_t pool_idx, cnt_idx;
 271         uint32_t i;
 272         int ret;
 273
 274         if (shared && !mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) &&
 275             data.dword)
 276                 return data.dword;
 277         for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 278                 pool = cmng->pools[pool_idx];
 279                 if (!pool)
 280                         continue;
 281                 cnt = TAILQ_FIRST(&pool->counters[0]);
 282                 if (cnt)
 283                         break;
 284         }
 285         if (!cnt) {
 286                 struct mlx5_flow_counter_pool **pools;
 287                 uint32_t size;
 288
 289                 if (n_valid == cmng->n) {
 290                         /* Resize the container pool array. */
 291                         size = sizeof(struct mlx5_flow_counter_pool *) *
 292                                      (n_valid + MLX5_CNT_CONTAINER_RESIZE);
 293                         pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0,
 294                                             SOCKET_ID_ANY);
 295                         if (!pools)
 296                                 return 0;
 297                         if (n_valid) {
 298                                 memcpy(pools, cmng->pools,
 299                                        sizeof(struct mlx5_flow_counter_pool *) *
 300                                        n_valid);
 301                                 mlx5_free(cmng->pools);
 302                         }
 303                         cmng->pools = pools;
 304                         cmng->n += MLX5_CNT_CONTAINER_RESIZE;
 305                 }
 306                 /* Allocate memory for new pool*/
 307                 size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL;
 308                 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
 309                 if (!pool)
 310                         return 0;
 311                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
 312                         cnt = MLX5_POOL_GET_CNT(pool, i);
 313                         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 314                 }
 315                 cnt = MLX5_POOL_GET_CNT(pool, 0);
 316                 cmng->pools[n_valid] = pool;
 317                 pool_idx = n_valid;
 318                 cmng->n_valid++;
 319         }
 320         TAILQ_REMOVE(&pool->counters[0], cnt, next);
 321         i = MLX5_CNT_ARRAY_IDX(pool, cnt);
 322         cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i);
 323         if (shared) {
 324                 data.dword = cnt_idx;
 325                 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
 326                         return 0;
 327                 cnt->shared_info.id = id;
 328                 cnt_idx |= MLX5_CNT_SHARED_OFFSET;
 329         }
 330         /* Create counter with Verbs. */
 331         ret = flow_verbs_counter_create(dev, cnt);
 332         if (!ret) {
 333                 cnt->dcs_when_active = cnt->dcs_when_free;
 334                 cnt->hits = 0;
 335                 cnt->bytes = 0;
 336                 return cnt_idx;
 337         }
 338         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 339         /* Some error occurred in Verbs library. */
 340         rte_errno = -ret;
 341         return 0;
 342 }
 343
 344 /**
 345  * Release a flow counter.
 346  *
 347  * @param[in] dev
 348  *   Pointer to the Ethernet device structure.
 349  * @param[in] counter
 350  *   Index to the counter handler.
 351  */
 352 static void
 353 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
 354 {
 355         struct mlx5_priv *priv = dev->data->dev_private;
 356         struct mlx5_flow_counter_pool *pool;
 357         struct mlx5_flow_counter *cnt;
 358
 359         cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool);
 360         if (IS_LEGACY_SHARED_CNT(counter) &&
 361             mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
 362                 return;
 363 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 364         claim_zero(mlx5_glue->destroy_counter_set
 365                         ((struct ibv_counter_set *)cnt->dcs_when_active));
 366 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 367         claim_zero(mlx5_glue->destroy_counters
 368                                 ((struct ibv_counters *)cnt->dcs_when_active));
 369 #endif
 370         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 371 }
 372
 373 /**
 374  * Query a flow counter via Verbs library call.
 375  *
 376  * @see rte_flow_query()
 377  * @see rte_flow_ops
 378  */
 379 static int
 380 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 381                          struct rte_flow *flow, void *data,
 382                          struct rte_flow_error *error)
 383 {
 384 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 385         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 386         if (flow->counter) {
 387                 struct mlx5_flow_counter_pool *pool;
 388                 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
 389                                                 (dev, flow->counter, &pool);
 390                 struct rte_flow_query_count *qc = data;
 391                 uint64_t counters[2] = {0, 0};
 392 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 393                 struct ibv_query_counter_set_attr query_cs_attr = {
 394                         .dcs_when_free = (struct ibv_counter_set *)
 395                                                 cnt->dcs_when_active,
 396                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 397                 };
 398                 struct ibv_counter_set_data query_out = {
 399                         .out = counters,
 400                         .outlen = 2 * sizeof(uint64_t),
 401                 };
 402                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 403                                                        &query_out);
 404 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 405                 int err = mlx5_glue->query_counters
 406                         ((struct ibv_counters *)cnt->dcs_when_active, counters,
 407                                 RTE_DIM(counters),
 408                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 409 #endif
 410                 if (err)
 411                         return rte_flow_error_set
 412                                 (error, err,
 413                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 414                                  NULL,
 415                                  "cannot read counter");
 416                 qc->hits_set = 1;
 417                 qc->bytes_set = 1;
 418                 qc->hits = counters[0] - cnt->hits;
 419                 qc->bytes = counters[1] - cnt->bytes;
 420                 if (qc->reset) {
 421                         cnt->hits = counters[0];
 422                         cnt->bytes = counters[1];
 423                 }
 424                 return 0;
 425         }
 426         return rte_flow_error_set(error, EINVAL,
 427                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 428                                   NULL,
 429                                   "flow does not have counter");
 430 #else
 431         (void)flow;
 432         (void)data;
 433         return rte_flow_error_set(error, ENOTSUP,
 434                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 435                                   NULL,
 436                                   "counters are not available");
 437 #endif
 438 }
 439
 440 /**
 441  * Add a verbs item specification into @p verbs.
 442  *
 443  * @param[out] verbs
 444  *   Pointer to verbs structure.
 445  * @param[in] src
 446  *   Create specification.
 447  * @param[in] size
 448  *   Size in bytes of the specification to copy.
 449  */
 450 static void
 451 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
 452                     void *src, unsigned int size)
 453 {
 454         void *dst;
 455
 456         if (!verbs)
 457                 return;
 458         MLX5_ASSERT(verbs->specs);
 459         dst = (void *)(verbs->specs + verbs->size);
 460         memcpy(dst, src, size);
 461         ++verbs->attr.num_of_specs;
 462         verbs->size += size;
 463 }
 464
 465 /**
 466  * Convert the @p item into a Verbs specification. This function assumes that
 467  * the input is valid and that there is space to insert the requested item
 468  * into the flow.
 469  *
 470  * @param[in, out] dev_flow
 471  *   Pointer to dev_flow structure.
 472  * @param[in] item
 473  *   Item specification.
 474  * @param[in] item_flags
 475  *   Parsed item flags.
 476  */
 477 static void
 478 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 479                               const struct rte_flow_item *item,
 480                               uint64_t item_flags)
 481 {
 482         const struct rte_flow_item_eth *spec = item->spec;
 483         const struct rte_flow_item_eth *mask = item->mask;
 484         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 485         struct ibv_flow_spec_eth eth = {
 486                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 487                 .size = size,
 488         };
 489
 490         if (!mask)
 491                 mask = &rte_flow_item_eth_mask;
 492         if (spec) {
 493                 unsigned int i;
 494
 495                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 496                         RTE_ETHER_ADDR_LEN);
 497                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 498                         RTE_ETHER_ADDR_LEN);
 499                 eth.val.ether_type = spec->type;
 500                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 501                         RTE_ETHER_ADDR_LEN);
 502                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 503                         RTE_ETHER_ADDR_LEN);
 504                 eth.mask.ether_type = mask->type;
 505                 /* Remove unwanted bits from values. */
 506                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 507                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 508                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 509                 }
 510                 eth.val.ether_type &= eth.mask.ether_type;
 511         }
 512         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 513 }
 514
 515 /**
 516  * Update the VLAN tag in the Verbs Ethernet specification.
 517  * This function assumes that the input is valid and there is space to add
 518  * the requested item.
 519  *
 520  * @param[in, out] attr
 521  *   Pointer to Verbs attributes structure.
 522  * @param[in] eth
 523  *   Verbs structure containing the VLAN information to copy.
 524  */
 525 static void
 526 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 527                             struct ibv_flow_spec_eth *eth)
 528 {
 529         unsigned int i;
 530         const enum ibv_flow_spec_type search = eth->type;
 531         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 532                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 533
 534         for (i = 0; i != attr->num_of_specs; ++i) {
 535                 if (hdr->type == search) {
 536                         struct ibv_flow_spec_eth *e =
 537                                 (struct ibv_flow_spec_eth *)hdr;
 538
 539                         e->val.vlan_tag = eth->val.vlan_tag;
 540                         e->mask.vlan_tag = eth->mask.vlan_tag;
 541                         e->val.ether_type = eth->val.ether_type;
 542                         e->mask.ether_type = eth->mask.ether_type;
 543                         break;
 544                 }
 545                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 546         }
 547 }
 548
 549 /**
 550  * Convert the @p item into a Verbs specification. This function assumes that
 551  * the input is valid and that there is space to insert the requested item
 552  * into the flow.
 553  *
 554  * @param[in, out] dev_flow
 555  *   Pointer to dev_flow structure.
 556  * @param[in] item
 557  *   Item specification.
 558  * @param[in] item_flags
 559  *   Parsed item flags.
 560  */
 561 static void
 562 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 563                                const struct rte_flow_item *item,
 564                                uint64_t item_flags)
 565 {
 566         const struct rte_flow_item_vlan *spec = item->spec;
 567         const struct rte_flow_item_vlan *mask = item->mask;
 568         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 569         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 570         struct ibv_flow_spec_eth eth = {
 571                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 572                 .size = size,
 573         };
 574         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 575                                       MLX5_FLOW_LAYER_OUTER_L2;
 576
 577         if (!mask)
 578                 mask = &rte_flow_item_vlan_mask;
 579         if (spec) {
 580                 eth.val.vlan_tag = spec->tci;
 581                 eth.mask.vlan_tag = mask->tci;
 582                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 583                 eth.val.ether_type = spec->inner_type;
 584                 eth.mask.ether_type = mask->inner_type;
 585                 eth.val.ether_type &= eth.mask.ether_type;
 586         }
 587         if (!(item_flags & l2m))
 588                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 589         else
 590                 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, &eth);
 591         if (!tunnel)
 592                 dev_flow->handle->vf_vlan.tag =
 593                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 594 }
 595
 596 /**
 597  * Convert the @p item into a Verbs specification. This function assumes that
 598  * the input is valid and that there is space to insert the requested item
 599  * into the flow.
 600  *
 601  * @param[in, out] dev_flow
 602  *   Pointer to dev_flow structure.
 603  * @param[in] item
 604  *   Item specification.
 605  * @param[in] item_flags
 606  *   Parsed item flags.
 607  */
 608 static void
 609 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 610                                const struct rte_flow_item *item,
 611                                uint64_t item_flags)
 612 {
 613         const struct rte_flow_item_ipv4 *spec = item->spec;
 614         const struct rte_flow_item_ipv4 *mask = item->mask;
 615         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 616         struct ibv_flow_spec_ipv4_ext ipv4 = {
 617                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 618                 .size = size,
 619         };
 620
 621         if (!mask)
 622                 mask = &rte_flow_item_ipv4_mask;
 623         if (spec) {
 624                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 625                         .src_ip = spec->hdr.src_addr,
 626                         .dst_ip = spec->hdr.dst_addr,
 627                         .proto = spec->hdr.next_proto_id,
 628                         .tos = spec->hdr.type_of_service,
 629                 };
 630                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 631                         .src_ip = mask->hdr.src_addr,
 632                         .dst_ip = mask->hdr.dst_addr,
 633                         .proto = mask->hdr.next_proto_id,
 634                         .tos = mask->hdr.type_of_service,
 635                 };
 636                 /* Remove unwanted bits from values. */
 637                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 638                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 639                 ipv4.val.proto &= ipv4.mask.proto;
 640                 ipv4.val.tos &= ipv4.mask.tos;
 641         }
 642         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 643 }
 644
 645 /**
 646  * Convert the @p item into a Verbs specification. This function assumes that
 647  * the input is valid and that there is space to insert the requested item
 648  * into the flow.
 649  *
 650  * @param[in, out] dev_flow
 651  *   Pointer to dev_flow structure.
 652  * @param[in] item
 653  *   Item specification.
 654  * @param[in] item_flags
 655  *   Parsed item flags.
 656  */
 657 static void
 658 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 659                                const struct rte_flow_item *item,
 660                                uint64_t item_flags)
 661 {
 662         const struct rte_flow_item_ipv6 *spec = item->spec;
 663         const struct rte_flow_item_ipv6 *mask = item->mask;
 664         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 665         struct ibv_flow_spec_ipv6 ipv6 = {
 666                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 667                 .size = size,
 668         };
 669
 670         if (!mask)
 671                 mask = &rte_flow_item_ipv6_mask;
 672         if (spec) {
 673                 unsigned int i;
 674                 uint32_t vtc_flow_val;
 675                 uint32_t vtc_flow_mask;
 676
 677                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 678                        RTE_DIM(ipv6.val.src_ip));
 679                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 680                        RTE_DIM(ipv6.val.dst_ip));
 681                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 682                        RTE_DIM(ipv6.mask.src_ip));
 683                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 684                        RTE_DIM(ipv6.mask.dst_ip));
 685                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 686                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 687                 ipv6.val.flow_label =
 688                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 689                                          RTE_IPV6_HDR_FL_SHIFT);
 690                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 691                                          RTE_IPV6_HDR_TC_SHIFT;
 692                 ipv6.val.next_hdr = spec->hdr.proto;
 693                 ipv6.mask.flow_label =
 694                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 695                                          RTE_IPV6_HDR_FL_SHIFT);
 696                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 697                                           RTE_IPV6_HDR_TC_SHIFT;
 698                 ipv6.mask.next_hdr = mask->hdr.proto;
 699                 /* Remove unwanted bits from values. */
 700                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 701                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 702                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 703                 }
 704                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 705                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 706                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 707         }
 708         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 709 }
 710
 711 /**
 712  * Convert the @p item into a Verbs specification. This function assumes that
 713  * the input is valid and that there is space to insert the requested item
 714  * into the flow.
 715  *
 716  * @param[in, out] dev_flow
 717  *   Pointer to dev_flow structure.
 718  * @param[in] item
 719  *   Item specification.
 720  * @param[in] item_flags
 721  *   Parsed item flags.
 722  */
 723 static void
 724 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 725                               const struct rte_flow_item *item,
 726                               uint64_t item_flags __rte_unused)
 727 {
 728         const struct rte_flow_item_tcp *spec = item->spec;
 729         const struct rte_flow_item_tcp *mask = item->mask;
 730         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 731         struct ibv_flow_spec_tcp_udp tcp = {
 732                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 733                 .size = size,
 734         };
 735
 736         if (!mask)
 737                 mask = &rte_flow_item_tcp_mask;
 738         if (spec) {
 739                 tcp.val.dst_port = spec->hdr.dst_port;
 740                 tcp.val.src_port = spec->hdr.src_port;
 741                 tcp.mask.dst_port = mask->hdr.dst_port;
 742                 tcp.mask.src_port = mask->hdr.src_port;
 743                 /* Remove unwanted bits from values. */
 744                 tcp.val.src_port &= tcp.mask.src_port;
 745                 tcp.val.dst_port &= tcp.mask.dst_port;
 746         }
 747         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 748 }
 749
 750 /**
 751  * Convert the @p item into a Verbs specification. This function assumes that
 752  * the input is valid and that there is space to insert the requested item
 753  * into the flow.
 754  *
 755  * @param[in, out] dev_flow
 756  *   Pointer to dev_flow structure.
 757  * @param[in] item
 758  *   Item specification.
 759  * @param[in] item_flags
 760  *   Parsed item flags.
 761  */
 762 static void
 763 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 764                               const struct rte_flow_item *item,
 765                               uint64_t item_flags __rte_unused)
 766 {
 767         const struct rte_flow_item_udp *spec = item->spec;
 768         const struct rte_flow_item_udp *mask = item->mask;
 769         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 770         struct ibv_flow_spec_tcp_udp udp = {
 771                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 772                 .size = size,
 773         };
 774
 775         if (!mask)
 776                 mask = &rte_flow_item_udp_mask;
 777         if (spec) {
 778                 udp.val.dst_port = spec->hdr.dst_port;
 779                 udp.val.src_port = spec->hdr.src_port;
 780                 udp.mask.dst_port = mask->hdr.dst_port;
 781                 udp.mask.src_port = mask->hdr.src_port;
 782                 /* Remove unwanted bits from values. */
 783                 udp.val.src_port &= udp.mask.src_port;
 784                 udp.val.dst_port &= udp.mask.dst_port;
 785         }
 786         item++;
 787         while (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 788                 item++;
 789         if (!(udp.val.dst_port & udp.mask.dst_port)) {
 790                 switch ((item)->type) {
 791                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 792                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN);
 793                         udp.mask.dst_port = 0xffff;
 794                         break;
 795                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 796                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE);
 797                         udp.mask.dst_port = 0xffff;
 798                         break;
 799                 case RTE_FLOW_ITEM_TYPE_MPLS:
 800                         udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS);
 801                         udp.mask.dst_port = 0xffff;
 802                         break;
 803                 default:
 804                         break;
 805                 }
 806         }
 807
 808         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 809 }
 810
 811 /**
 812  * Convert the @p item into a Verbs specification. This function assumes that
 813  * the input is valid and that there is space to insert the requested item
 814  * into the flow.
 815  *
 816  * @param[in, out] dev_flow
 817  *   Pointer to dev_flow structure.
 818  * @param[in] item
 819  *   Item specification.
 820  * @param[in] item_flags
 821  *   Parsed item flags.
 822  */
 823 static void
 824 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 825                                 const struct rte_flow_item *item,
 826                                 uint64_t item_flags __rte_unused)
 827 {
 828         const struct rte_flow_item_vxlan *spec = item->spec;
 829         const struct rte_flow_item_vxlan *mask = item->mask;
 830         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 831         struct ibv_flow_spec_tunnel vxlan = {
 832                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 833                 .size = size,
 834         };
 835         union vni {
 836                 uint32_t vlan_id;
 837                 uint8_t vni[4];
 838         } id = { .vlan_id = 0, };
 839
 840         if (!mask)
 841                 mask = &rte_flow_item_vxlan_mask;
 842         if (spec) {
 843                 memcpy(&id.vni[1], spec->vni, 3);
 844                 vxlan.val.tunnel_id = id.vlan_id;
 845                 memcpy(&id.vni[1], mask->vni, 3);
 846                 vxlan.mask.tunnel_id = id.vlan_id;
 847                 /* Remove unwanted bits from values. */
 848                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 849         }
 850         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 851 }
 852
 853 /**
 854  * Convert the @p item into a Verbs specification. This function assumes that
 855  * the input is valid and that there is space to insert the requested item
 856  * into the flow.
 857  *
 858  * @param[in, out] dev_flow
 859  *   Pointer to dev_flow structure.
 860  * @param[in] item
 861  *   Item specification.
 862  * @param[in] item_flags
 863  *   Parsed item flags.
 864  */
 865 static void
 866 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 867                                     const struct rte_flow_item *item,
 868                                     uint64_t item_flags __rte_unused)
 869 {
 870         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 871         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 872         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 873         struct ibv_flow_spec_tunnel vxlan_gpe = {
 874                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 875                 .size = size,
 876         };
 877         union vni {
 878                 uint32_t vlan_id;
 879                 uint8_t vni[4];
 880         } id = { .vlan_id = 0, };
 881
 882         if (!mask)
 883                 mask = &rte_flow_item_vxlan_gpe_mask;
 884         if (spec) {
 885                 memcpy(&id.vni[1], spec->vni, 3);
 886                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 887                 memcpy(&id.vni[1], mask->vni, 3);
 888                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 889                 /* Remove unwanted bits from values. */
 890                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 891         }
 892         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 893 }
 894
 895 /**
 896  * Update the protocol in Verbs IPv4/IPv6 spec.
 897  *
 898  * @param[in, out] attr
 899  *   Pointer to Verbs attributes structure.
 900  * @param[in] search
 901  *   Specification type to search in order to update the IP protocol.
 902  * @param[in] protocol
 903  *   Protocol value to set if none is present in the specification.
 904  */
 905 static void
 906 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 907                                        enum ibv_flow_spec_type search,
 908                                        uint8_t protocol)
 909 {
 910         unsigned int i;
 911         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 912                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 913
 914         if (!attr)
 915                 return;
 916         for (i = 0; i != attr->num_of_specs; ++i) {
 917                 if (hdr->type == search) {
 918                         union {
 919                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 920                                 struct ibv_flow_spec_ipv6 *ipv6;
 921                         } ip;
 922
 923                         switch (search) {
 924                         case IBV_FLOW_SPEC_IPV4_EXT:
 925                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 926                                 if (!ip.ipv4->val.proto) {
 927                                         ip.ipv4->val.proto = protocol;
 928                                         ip.ipv4->mask.proto = 0xff;
 929                                 }
 930                                 break;
 931                         case IBV_FLOW_SPEC_IPV6:
 932                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 933                                 if (!ip.ipv6->val.next_hdr) {
 934                                         ip.ipv6->val.next_hdr = protocol;
 935                                         ip.ipv6->mask.next_hdr = 0xff;
 936                                 }
 937                                 break;
 938                         default:
 939                                 break;
 940                         }
 941                         break;
 942                 }
 943                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 944         }
 945 }
 946
 947 /**
 948  * Convert the @p item into a Verbs specification. This function assumes that
 949  * the input is valid and that there is space to insert the requested item
 950  * into the flow.
 951  *
 952  * @param[in, out] dev_flow
 953  *   Pointer to dev_flow structure.
 954  * @param[in] item
 955  *   Item specification.
 956  * @param[in] item_flags
 957  *   Parsed item flags.
 958  */
 959 static void
 960 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 961                               const struct rte_flow_item *item __rte_unused,
 962                               uint64_t item_flags)
 963 {
 964         struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
 965 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 966         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 967         struct ibv_flow_spec_tunnel tunnel = {
 968                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 969                 .size = size,
 970         };
 971 #else
 972         const struct rte_flow_item_gre *spec = item->spec;
 973         const struct rte_flow_item_gre *mask = item->mask;
 974         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 975         struct ibv_flow_spec_gre tunnel = {
 976                 .type = IBV_FLOW_SPEC_GRE,
 977                 .size = size,
 978         };
 979
 980         if (!mask)
 981                 mask = &rte_flow_item_gre_mask;
 982         if (spec) {
 983                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 984                 tunnel.val.protocol = spec->protocol;
 985                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 986                 tunnel.mask.protocol = mask->protocol;
 987                 /* Remove unwanted bits from values. */
 988                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 989                 tunnel.val.protocol &= tunnel.mask.protocol;
 990                 tunnel.val.key &= tunnel.mask.key;
 991         }
 992 #endif
 993         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 994                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 995                                                        IBV_FLOW_SPEC_IPV4_EXT,
 996                                                        IPPROTO_GRE);
 997         else
 998                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 999                                                        IBV_FLOW_SPEC_IPV6,
1000                                                        IPPROTO_GRE);
1001         flow_verbs_spec_add(verbs, &tunnel, size);
1002 }
1003
1004 /**
1005  * Convert the @p action into a Verbs specification. This function assumes that
1006  * the input is valid and that there is space to insert the requested action
1007  * into the flow. This function also return the action that was added.
1008  *
1009  * @param[in, out] dev_flow
1010  *   Pointer to dev_flow structure.
1011  * @param[in] item
1012  *   Item specification.
1013  * @param[in] item_flags
1014  *   Parsed item flags.
1015  */
1016 static void
1017 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
1018                                const struct rte_flow_item *item __rte_unused,
1019                                uint64_t item_flags __rte_unused)
1020 {
1021 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1022         const struct rte_flow_item_mpls *spec = item->spec;
1023         const struct rte_flow_item_mpls *mask = item->mask;
1024         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
1025         struct ibv_flow_spec_mpls mpls = {
1026                 .type = IBV_FLOW_SPEC_MPLS,
1027                 .size = size,
1028         };
1029
1030         if (!mask)
1031                 mask = &rte_flow_item_mpls_mask;
1032         if (spec) {
1033                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
1034                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
1035                 /* Remove unwanted bits from values.  */
1036                 mpls.val.label &= mpls.mask.label;
1037         }
1038         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
1039 #endif
1040 }
1041
1042 /**
1043  * Convert the @p action into a Verbs specification. This function assumes that
1044  * the input is valid and that there is space to insert the requested action
1045  * into the flow.
1046  *
1047  * @param[in] dev_flow
1048  *   Pointer to mlx5_flow.
1049  * @param[in] action
1050  *   Action configuration.
1051  */
1052 static void
1053 flow_verbs_translate_action_drop
1054         (struct mlx5_flow *dev_flow,
1055          const struct rte_flow_action *action __rte_unused)
1056 {
1057         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1058         struct ibv_flow_spec_action_drop drop = {
1059                         .type = IBV_FLOW_SPEC_ACTION_DROP,
1060                         .size = size,
1061         };
1062
1063         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
1064 }
1065
1066 /**
1067  * Convert the @p action into a Verbs specification. This function assumes that
1068  * the input is valid and that there is space to insert the requested action
1069  * into the flow.
1070  *
1071  * @param[in] rss_desc
1072  *   Pointer to mlx5_flow_rss_desc.
1073  * @param[in] action
1074  *   Action configuration.
1075  */
1076 static void
1077 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc,
1078                                   const struct rte_flow_action *action)
1079 {
1080         const struct rte_flow_action_queue *queue = action->conf;
1081
1082         rss_desc->queue[0] = queue->index;
1083         rss_desc->queue_num = 1;
1084 }
1085
1086 /**
1087  * Convert the @p action into a Verbs specification. This function assumes that
1088  * the input is valid and that there is space to insert the requested action
1089  * into the flow.
1090  *
1091  * @param[in] rss_desc
1092  *   Pointer to mlx5_flow_rss_desc.
1093  * @param[in] action
1094  *   Action configuration.
1095  */
1096 static void
1097 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc,
1098                                 const struct rte_flow_action *action)
1099 {
1100         const struct rte_flow_action_rss *rss = action->conf;
1101         const uint8_t *rss_key;
1102
1103         memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t));
1104         rss_desc->queue_num = rss->queue_num;
1105         /* NULL RSS key indicates default RSS key. */
1106         rss_key = !rss->key ? rss_hash_default_key : rss->key;
1107         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
1108         /*
1109          * rss->level and rss.types should be set in advance when expanding
1110          * items for RSS.
1111          */
1112 }
1113
1114 /**
1115  * Convert the @p action into a Verbs specification. This function assumes that
1116  * the input is valid and that there is space to insert the requested action
1117  * into the flow.
1118  *
1119  * @param[in] dev_flow
1120  *   Pointer to mlx5_flow.
1121  * @param[in] action
1122  *   Action configuration.
1123  */
1124 static void
1125 flow_verbs_translate_action_flag
1126         (struct mlx5_flow *dev_flow,
1127          const struct rte_flow_action *action __rte_unused)
1128 {
1129         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1130         struct ibv_flow_spec_action_tag tag = {
1131                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1132                 .size = size,
1133                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1134         };
1135
1136         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1137 }
1138
1139 /**
1140  * Convert the @p action into a Verbs specification. This function assumes that
1141  * the input is valid and that there is space to insert the requested action
1142  * into the flow.
1143  *
1144  * @param[in] dev_flow
1145  *   Pointer to mlx5_flow.
1146  * @param[in] action
1147  *   Action configuration.
1148  */
1149 static void
1150 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1151                                  const struct rte_flow_action *action)
1152 {
1153         const struct rte_flow_action_mark *mark = action->conf;
1154         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1155         struct ibv_flow_spec_action_tag tag = {
1156                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1157                 .size = size,
1158                 .tag_id = mlx5_flow_mark_set(mark->id),
1159         };
1160
1161         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1162 }
1163
1164 /**
1165  * Convert the @p action into a Verbs specification. This function assumes that
1166  * the input is valid and that there is space to insert the requested action
1167  * into the flow.
1168  *
1169  * @param[in] dev
1170  *   Pointer to the Ethernet device structure.
1171  * @param[in] action
1172  *   Action configuration.
1173  * @param[in] dev_flow
1174  *   Pointer to mlx5_flow.
1175  * @param[out] error
1176  *   Pointer to error structure.
1177  *
1178  * @return
1179  *   0 On success else a negative errno value is returned and rte_errno is set.
1180  */
1181 static int
1182 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1183                                   const struct rte_flow_action *action,
1184                                   struct rte_eth_dev *dev,
1185                                   struct rte_flow_error *error)
1186 {
1187         const struct rte_flow_action_count *count = action->conf;
1188         struct rte_flow *flow = dev_flow->flow;
1189 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1190         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1191         struct mlx5_flow_counter_pool *pool;
1192         struct mlx5_flow_counter *cnt = NULL;
1193         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1194         struct ibv_flow_spec_counter_action counter = {
1195                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1196                 .size = size,
1197         };
1198 #endif
1199
1200         if (!flow->counter) {
1201                 flow->counter = flow_verbs_counter_new(dev, count->shared,
1202                                                        count->id);
1203                 if (!flow->counter)
1204                         return rte_flow_error_set(error, rte_errno,
1205                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1206                                                   action,
1207                                                   "cannot get counter"
1208                                                   " context.");
1209         }
1210 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1211         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1212         counter.counter_set_handle =
1213                 ((struct ibv_counter_set *)cnt->dcs_when_active)->handle;
1214         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1215 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1216         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1217         counter.counters = (struct ibv_counters *)cnt->dcs_when_active;
1218         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1219 #endif
1220         return 0;
1221 }
1222
1223 /**
1224  * Internal validation function. For validating both actions and items.
1225  *
1226  * @param[in] dev
1227  *   Pointer to the Ethernet device structure.
1228  * @param[in] attr
1229  *   Pointer to the flow attributes.
1230  * @param[in] items
1231  *   Pointer to the list of items.
1232  * @param[in] actions
1233  *   Pointer to the list of actions.
1234  * @param[in] external
1235  *   This flow rule is created by request external to PMD.
1236  * @param[in] hairpin
1237  *   Number of hairpin TX actions, 0 means classic flow.
1238  * @param[out] error
1239  *   Pointer to the error structure.
1240  *
1241  * @return
1242  *   0 on success, a negative errno value otherwise and rte_errno is set.
1243  */
1244 static int
1245 flow_verbs_validate(struct rte_eth_dev *dev,
1246                     const struct rte_flow_attr *attr,
1247                     const struct rte_flow_item items[],
1248                     const struct rte_flow_action actions[],
1249                     bool external __rte_unused,
1250                     int hairpin __rte_unused,
1251                     struct rte_flow_error *error)
1252 {
1253         int ret;
1254         uint64_t action_flags = 0;
1255         uint64_t item_flags = 0;
1256         uint64_t last_item = 0;
1257         uint8_t next_protocol = 0xff;
1258         uint16_t ether_type = 0;
1259         bool is_empty_vlan = false;
1260
1261         if (items == NULL)
1262                 return -1;
1263         ret = mlx5_flow_validate_attributes(dev, attr, error);
1264         if (ret < 0)
1265                 return ret;
1266         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1267                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1268                 int ret = 0;
1269
1270                 switch (items->type) {
1271                 case RTE_FLOW_ITEM_TYPE_VOID:
1272                         break;
1273                 case RTE_FLOW_ITEM_TYPE_ETH:
1274                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1275                                                           false, error);
1276                         if (ret < 0)
1277                                 return ret;
1278                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1279                                              MLX5_FLOW_LAYER_OUTER_L2;
1280                         if (items->mask != NULL && items->spec != NULL) {
1281                                 ether_type =
1282                                         ((const struct rte_flow_item_eth *)
1283                                          items->spec)->type;
1284                                 ether_type &=
1285                                         ((const struct rte_flow_item_eth *)
1286                                          items->mask)->type;
1287                                 if (ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN))
1288                                         is_empty_vlan = true;
1289                                 ether_type = rte_be_to_cpu_16(ether_type);
1290                         } else {
1291                                 ether_type = 0;
1292                         }
1293                         break;
1294                 case RTE_FLOW_ITEM_TYPE_VLAN:
1295                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1296                                                            dev, error);
1297                         if (ret < 0)
1298                                 return ret;
1299                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1300                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1301                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1302                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1303                         if (items->mask != NULL && items->spec != NULL) {
1304                                 ether_type =
1305                                         ((const struct rte_flow_item_vlan *)
1306                                          items->spec)->inner_type;
1307                                 ether_type &=
1308                                         ((const struct rte_flow_item_vlan *)
1309                                          items->mask)->inner_type;
1310                                 ether_type = rte_be_to_cpu_16(ether_type);
1311                         } else {
1312                                 ether_type = 0;
1313                         }
1314                         is_empty_vlan = false;
1315                         break;
1316                 case RTE_FLOW_ITEM_TYPE_IPV4:
1317                         ret = mlx5_flow_validate_item_ipv4
1318                                                 (items, item_flags,
1319                                                  last_item, ether_type, NULL,
1320                                                  MLX5_ITEM_RANGE_NOT_ACCEPTED,
1321                                                  error);
1322                         if (ret < 0)
1323                                 return ret;
1324                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1325                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1326                         if (items->mask != NULL &&
1327                             ((const struct rte_flow_item_ipv4 *)
1328                              items->mask)->hdr.next_proto_id) {
1329                                 next_protocol =
1330                                         ((const struct rte_flow_item_ipv4 *)
1331                                          (items->spec))->hdr.next_proto_id;
1332                                 next_protocol &=
1333                                         ((const struct rte_flow_item_ipv4 *)
1334                                          (items->mask))->hdr.next_proto_id;
1335                         } else {
1336                                 /* Reset for inner layer. */
1337                                 next_protocol = 0xff;
1338                         }
1339                         break;
1340                 case RTE_FLOW_ITEM_TYPE_IPV6:
1341                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1342                                                            last_item,
1343                                                            ether_type, NULL,
1344                                                            error);
1345                         if (ret < 0)
1346                                 return ret;
1347                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1348                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1349                         if (items->mask != NULL &&
1350                             ((const struct rte_flow_item_ipv6 *)
1351                              items->mask)->hdr.proto) {
1352                                 next_protocol =
1353                                         ((const struct rte_flow_item_ipv6 *)
1354                                          items->spec)->hdr.proto;
1355                                 next_protocol &=
1356                                         ((const struct rte_flow_item_ipv6 *)
1357                                          items->mask)->hdr.proto;
1358                         } else {
1359                                 /* Reset for inner layer. */
1360                                 next_protocol = 0xff;
1361                         }
1362                         break;
1363                 case RTE_FLOW_ITEM_TYPE_UDP:
1364                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1365                                                           next_protocol,
1366                                                           error);
1367                         if (ret < 0)
1368                                 return ret;
1369                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1370                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1371                         break;
1372                 case RTE_FLOW_ITEM_TYPE_TCP:
1373                         ret = mlx5_flow_validate_item_tcp
1374                                                 (items, item_flags,
1375                                                  next_protocol,
1376                                                  &rte_flow_item_tcp_mask,
1377                                                  error);
1378                         if (ret < 0)
1379                                 return ret;
1380                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1381                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1382                         break;
1383                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1384                         ret = mlx5_flow_validate_item_vxlan(dev, items,
1385                                                             item_flags, attr,
1386                                                             error);
1387                         if (ret < 0)
1388                                 return ret;
1389                         last_item = MLX5_FLOW_LAYER_VXLAN;
1390                         break;
1391                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1392                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1393                                                                 item_flags,
1394                                                                 dev, error);
1395                         if (ret < 0)
1396                                 return ret;
1397                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1398                         break;
1399                 case RTE_FLOW_ITEM_TYPE_GRE:
1400                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1401                                                           next_protocol, error);
1402                         if (ret < 0)
1403                                 return ret;
1404                         last_item = MLX5_FLOW_LAYER_GRE;
1405                         break;
1406                 case RTE_FLOW_ITEM_TYPE_MPLS:
1407                         ret = mlx5_flow_validate_item_mpls(dev, items,
1408                                                            item_flags,
1409                                                            last_item, error);
1410                         if (ret < 0)
1411                                 return ret;
1412                         last_item = MLX5_FLOW_LAYER_MPLS;
1413                         break;
1414                 case RTE_FLOW_ITEM_TYPE_ICMP:
1415                 case RTE_FLOW_ITEM_TYPE_ICMP6:
1416                         return rte_flow_error_set(error, ENOTSUP,
1417                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1418                                                   NULL, "ICMP/ICMP6 "
1419                                                   "item not supported");
1420                 default:
1421                         return rte_flow_error_set(error, ENOTSUP,
1422                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1423                                                   NULL, "item not supported");
1424                 }
1425                 item_flags |= last_item;
1426         }
1427         if (is_empty_vlan)
1428                 return rte_flow_error_set(error, ENOTSUP,
1429                                                  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1430                     "VLAN matching without vid specification is not supported");
1431         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1432                 switch (actions->type) {
1433                 case RTE_FLOW_ACTION_TYPE_VOID:
1434                         break;
1435                 case RTE_FLOW_ACTION_TYPE_FLAG:
1436                         ret = mlx5_flow_validate_action_flag(action_flags,
1437                                                              attr,
1438                                                              error);
1439                         if (ret < 0)
1440                                 return ret;
1441                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1442                         break;
1443                 case RTE_FLOW_ACTION_TYPE_MARK:
1444                         ret = mlx5_flow_validate_action_mark(actions,
1445                                                              action_flags,
1446                                                              attr,
1447                                                              error);
1448                         if (ret < 0)
1449                                 return ret;
1450                         action_flags |= MLX5_FLOW_ACTION_MARK;
1451                         break;
1452                 case RTE_FLOW_ACTION_TYPE_DROP:
1453                         ret = mlx5_flow_validate_action_drop(action_flags,
1454                                                              attr,
1455                                                              error);
1456                         if (ret < 0)
1457                                 return ret;
1458                         action_flags |= MLX5_FLOW_ACTION_DROP;
1459                         break;
1460                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1461                         ret = mlx5_flow_validate_action_queue(actions,
1462                                                               action_flags, dev,
1463                                                               attr,
1464                                                               error);
1465                         if (ret < 0)
1466                                 return ret;
1467                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1468                         break;
1469                 case RTE_FLOW_ACTION_TYPE_RSS:
1470                         ret = mlx5_flow_validate_action_rss(actions,
1471                                                             action_flags, dev,
1472                                                             attr, item_flags,
1473                                                             error);
1474                         if (ret < 0)
1475                                 return ret;
1476                         action_flags |= MLX5_FLOW_ACTION_RSS;
1477                         break;
1478                 case RTE_FLOW_ACTION_TYPE_COUNT:
1479                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1480                         if (ret < 0)
1481                                 return ret;
1482                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1483                         break;
1484                 default:
1485                         return rte_flow_error_set(error, ENOTSUP,
1486                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1487                                                   actions,
1488                                                   "action not supported");
1489                 }
1490         }
1491         /*
1492          * Validate the drop action mutual exclusion with other actions.
1493          * Drop action is mutually-exclusive with any other action, except for
1494          * Count action.
1495          */
1496         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1497             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1498                 return rte_flow_error_set(error, EINVAL,
1499                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1500                                           "Drop action is mutually-exclusive "
1501                                           "with any other action, except for "
1502                                           "Count action");
1503         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1504                 return rte_flow_error_set(error, EINVAL,
1505                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1506                                           "no fate action is found");
1507         return 0;
1508 }
1509
1510 /**
1511  * Calculate the required bytes that are needed for the action part of the verbs
1512  * flow.
1513  *
1514  * @param[in] actions
1515  *   Pointer to the list of actions.
1516  *
1517  * @return
1518  *   The size of the memory needed for all actions.
1519  */
1520 static int
1521 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1522 {
1523         int size = 0;
1524
1525         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1526                 switch (actions->type) {
1527                 case RTE_FLOW_ACTION_TYPE_VOID:
1528                         break;
1529                 case RTE_FLOW_ACTION_TYPE_FLAG:
1530                         size += sizeof(struct ibv_flow_spec_action_tag);
1531                         break;
1532                 case RTE_FLOW_ACTION_TYPE_MARK:
1533                         size += sizeof(struct ibv_flow_spec_action_tag);
1534                         break;
1535                 case RTE_FLOW_ACTION_TYPE_DROP:
1536                         size += sizeof(struct ibv_flow_spec_action_drop);
1537                         break;
1538                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1539                         break;
1540                 case RTE_FLOW_ACTION_TYPE_RSS:
1541                         break;
1542                 case RTE_FLOW_ACTION_TYPE_COUNT:
1543 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1544         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1545                         size += sizeof(struct ibv_flow_spec_counter_action);
1546 #endif
1547                         break;
1548                 default:
1549                         break;
1550                 }
1551         }
1552         return size;
1553 }
1554
1555 /**
1556  * Calculate the required bytes that are needed for the item part of the verbs
1557  * flow.
1558  *
1559  * @param[in] items
1560  *   Pointer to the list of items.
1561  *
1562  * @return
1563  *   The size of the memory needed for all items.
1564  */
1565 static int
1566 flow_verbs_get_items_size(const struct rte_flow_item items[])
1567 {
1568         int size = 0;
1569
1570         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1571                 switch (items->type) {
1572                 case RTE_FLOW_ITEM_TYPE_VOID:
1573                         break;
1574                 case RTE_FLOW_ITEM_TYPE_ETH:
1575                         size += sizeof(struct ibv_flow_spec_eth);
1576                         break;
1577                 case RTE_FLOW_ITEM_TYPE_VLAN:
1578                         size += sizeof(struct ibv_flow_spec_eth);
1579                         break;
1580                 case RTE_FLOW_ITEM_TYPE_IPV4:
1581                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1582                         break;
1583                 case RTE_FLOW_ITEM_TYPE_IPV6:
1584                         size += sizeof(struct ibv_flow_spec_ipv6);
1585                         break;
1586                 case RTE_FLOW_ITEM_TYPE_UDP:
1587                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1588                         break;
1589                 case RTE_FLOW_ITEM_TYPE_TCP:
1590                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1591                         break;
1592                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1593                         size += sizeof(struct ibv_flow_spec_tunnel);
1594                         break;
1595                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1596                         size += sizeof(struct ibv_flow_spec_tunnel);
1597                         break;
1598 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1599                 case RTE_FLOW_ITEM_TYPE_GRE:
1600                         size += sizeof(struct ibv_flow_spec_gre);
1601                         break;
1602                 case RTE_FLOW_ITEM_TYPE_MPLS:
1603                         size += sizeof(struct ibv_flow_spec_mpls);
1604                         break;
1605 #else
1606                 case RTE_FLOW_ITEM_TYPE_GRE:
1607                         size += sizeof(struct ibv_flow_spec_tunnel);
1608                         break;
1609 #endif
1610                 default:
1611                         break;
1612                 }
1613         }
1614         return size;
1615 }
1616
1617 /**
1618  * Internal preparation function. Allocate mlx5_flow with the required size.
1619  * The required size is calculate based on the actions and items. This function
1620  * also returns the detected actions and items for later use.
1621  *
1622  * @param[in] dev
1623  *   Pointer to Ethernet device.
1624  * @param[in] attr
1625  *   Pointer to the flow attributes.
1626  * @param[in] items
1627  *   Pointer to the list of items.
1628  * @param[in] actions
1629  *   Pointer to the list of actions.
1630  * @param[out] error
1631  *   Pointer to the error structure.
1632  *
1633  * @return
1634  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1635  *   is set.
1636  */
1637 static struct mlx5_flow *
1638 flow_verbs_prepare(struct rte_eth_dev *dev,
1639                    const struct rte_flow_attr *attr __rte_unused,
1640                    const struct rte_flow_item items[],
1641                    const struct rte_flow_action actions[],
1642                    struct rte_flow_error *error)
1643 {
1644         size_t size = 0;
1645         uint32_t handle_idx = 0;
1646         struct mlx5_flow *dev_flow;
1647         struct mlx5_flow_handle *dev_handle;
1648         struct mlx5_priv *priv = dev->data->dev_private;
1649         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1650
1651         MLX5_ASSERT(wks);
1652         size += flow_verbs_get_actions_size(actions);
1653         size += flow_verbs_get_items_size(items);
1654         if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1655                 rte_flow_error_set(error, E2BIG,
1656                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1657                                    "Verbs spec/action size too large");
1658                 return NULL;
1659         }
1660         /* In case of corrupting the memory. */
1661         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1662                 rte_flow_error_set(error, ENOSPC,
1663                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1664                                    "not free temporary device flow");
1665                 return NULL;
1666         }
1667         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1668                                    &handle_idx);
1669         if (!dev_handle) {
1670                 rte_flow_error_set(error, ENOMEM,
1671                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1672                                    "not enough memory to create flow handle");
1673                 return NULL;
1674         }
1675         MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows));
1676         dev_flow = &wks->flows[wks->flow_idx++];
1677         dev_flow->handle = dev_handle;
1678         dev_flow->handle_idx = handle_idx;
1679         /* Memcpy is used, only size needs to be cleared to 0. */
1680         dev_flow->verbs.size = 0;
1681         dev_flow->verbs.attr.num_of_specs = 0;
1682         dev_flow->ingress = attr->ingress;
1683         dev_flow->hash_fields = 0;
1684         /* Need to set transfer attribute: not supported in Verbs mode. */
1685         return dev_flow;
1686 }
1687
1688 /**
1689  * Fill the flow with verb spec.
1690  *
1691  * @param[in] dev
1692  *   Pointer to Ethernet device.
1693  * @param[in, out] dev_flow
1694  *   Pointer to the mlx5 flow.
1695  * @param[in] attr
1696  *   Pointer to the flow attributes.
1697  * @param[in] items
1698  *   Pointer to the list of items.
1699  * @param[in] actions
1700  *   Pointer to the list of actions.
1701  * @param[out] error
1702  *   Pointer to the error structure.
1703  *
1704  * @return
1705  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1706  */
1707 static int
1708 flow_verbs_translate(struct rte_eth_dev *dev,
1709                      struct mlx5_flow *dev_flow,
1710                      const struct rte_flow_attr *attr,
1711                      const struct rte_flow_item items[],
1712                      const struct rte_flow_action actions[],
1713                      struct rte_flow_error *error)
1714 {
1715         uint64_t item_flags = 0;
1716         uint64_t action_flags = 0;
1717         uint64_t priority = attr->priority;
1718         uint32_t subpriority = 0;
1719         struct mlx5_priv *priv = dev->data->dev_private;
1720         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1721         struct mlx5_flow_rss_desc *rss_desc;
1722
1723         MLX5_ASSERT(wks);
1724         rss_desc = &wks->rss_desc;
1725         if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
1726                 priority = priv->config.flow_prio - 1;
1727         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1728                 int ret;
1729
1730                 switch (actions->type) {
1731                 case RTE_FLOW_ACTION_TYPE_VOID:
1732                         break;
1733                 case RTE_FLOW_ACTION_TYPE_FLAG:
1734                         flow_verbs_translate_action_flag(dev_flow, actions);
1735                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1736                         dev_flow->handle->mark = 1;
1737                         break;
1738                 case RTE_FLOW_ACTION_TYPE_MARK:
1739                         flow_verbs_translate_action_mark(dev_flow, actions);
1740                         action_flags |= MLX5_FLOW_ACTION_MARK;
1741                         dev_flow->handle->mark = 1;
1742                         break;
1743                 case RTE_FLOW_ACTION_TYPE_DROP:
1744                         flow_verbs_translate_action_drop(dev_flow, actions);
1745                         action_flags |= MLX5_FLOW_ACTION_DROP;
1746                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
1747                         break;
1748                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1749                         flow_verbs_translate_action_queue(rss_desc, actions);
1750                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1751                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1752                         break;
1753                 case RTE_FLOW_ACTION_TYPE_RSS:
1754                         flow_verbs_translate_action_rss(rss_desc, actions);
1755                         action_flags |= MLX5_FLOW_ACTION_RSS;
1756                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1757                         break;
1758                 case RTE_FLOW_ACTION_TYPE_COUNT:
1759                         ret = flow_verbs_translate_action_count(dev_flow,
1760                                                                 actions,
1761                                                                 dev, error);
1762                         if (ret < 0)
1763                                 return ret;
1764                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1765                         break;
1766                 default:
1767                         return rte_flow_error_set(error, ENOTSUP,
1768                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1769                                                   actions,
1770                                                   "action not supported");
1771                 }
1772         }
1773         dev_flow->act_flags = action_flags;
1774         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1775                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1776
1777                 switch (items->type) {
1778                 case RTE_FLOW_ITEM_TYPE_VOID:
1779                         break;
1780                 case RTE_FLOW_ITEM_TYPE_ETH:
1781                         flow_verbs_translate_item_eth(dev_flow, items,
1782                                                       item_flags);
1783                         subpriority = MLX5_PRIORITY_MAP_L2;
1784                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1785                                                MLX5_FLOW_LAYER_OUTER_L2;
1786                         break;
1787                 case RTE_FLOW_ITEM_TYPE_VLAN:
1788                         flow_verbs_translate_item_vlan(dev_flow, items,
1789                                                        item_flags);
1790                         subpriority = MLX5_PRIORITY_MAP_L2;
1791                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1792                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1793                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1794                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1795                         break;
1796                 case RTE_FLOW_ITEM_TYPE_IPV4:
1797                         flow_verbs_translate_item_ipv4(dev_flow, items,
1798                                                        item_flags);
1799                         subpriority = MLX5_PRIORITY_MAP_L3;
1800                         dev_flow->hash_fields |=
1801                                 mlx5_flow_hashfields_adjust
1802                                         (rss_desc, tunnel,
1803                                          MLX5_IPV4_LAYER_TYPES,
1804                                          MLX5_IPV4_IBV_RX_HASH);
1805                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1806                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1807                         break;
1808                 case RTE_FLOW_ITEM_TYPE_IPV6:
1809                         flow_verbs_translate_item_ipv6(dev_flow, items,
1810                                                        item_flags);
1811                         subpriority = MLX5_PRIORITY_MAP_L3;
1812                         dev_flow->hash_fields |=
1813                                 mlx5_flow_hashfields_adjust
1814                                         (rss_desc, tunnel,
1815                                          MLX5_IPV6_LAYER_TYPES,
1816                                          MLX5_IPV6_IBV_RX_HASH);
1817                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1818                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1819                         break;
1820                 case RTE_FLOW_ITEM_TYPE_TCP:
1821                         flow_verbs_translate_item_tcp(dev_flow, items,
1822                                                       item_flags);
1823                         subpriority = MLX5_PRIORITY_MAP_L4;
1824                         dev_flow->hash_fields |=
1825                                 mlx5_flow_hashfields_adjust
1826                                         (rss_desc, tunnel, ETH_RSS_TCP,
1827                                          (IBV_RX_HASH_SRC_PORT_TCP |
1828                                           IBV_RX_HASH_DST_PORT_TCP));
1829                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1830                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1831                         break;
1832                 case RTE_FLOW_ITEM_TYPE_UDP:
1833                         flow_verbs_translate_item_udp(dev_flow, items,
1834                                                       item_flags);
1835                         subpriority = MLX5_PRIORITY_MAP_L4;
1836                         dev_flow->hash_fields |=
1837                                 mlx5_flow_hashfields_adjust
1838                                         (rss_desc, tunnel, ETH_RSS_UDP,
1839                                          (IBV_RX_HASH_SRC_PORT_UDP |
1840                                           IBV_RX_HASH_DST_PORT_UDP));
1841                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1842                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1843                         break;
1844                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1845                         flow_verbs_translate_item_vxlan(dev_flow, items,
1846                                                         item_flags);
1847                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1848                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1849                         break;
1850                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1851                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1852                                                             item_flags);
1853                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1854                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1855                         break;
1856                 case RTE_FLOW_ITEM_TYPE_GRE:
1857                         flow_verbs_translate_item_gre(dev_flow, items,
1858                                                       item_flags);
1859                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1860                         item_flags |= MLX5_FLOW_LAYER_GRE;
1861                         break;
1862                 case RTE_FLOW_ITEM_TYPE_MPLS:
1863                         flow_verbs_translate_item_mpls(dev_flow, items,
1864                                                        item_flags);
1865                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1866                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1867                         break;
1868                 default:
1869                         return rte_flow_error_set(error, ENOTSUP,
1870                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1871                                                   NULL, "item not supported");
1872                 }
1873         }
1874         dev_flow->handle->layers = item_flags;
1875         /* Other members of attr will be ignored. */
1876         dev_flow->verbs.attr.priority =
1877                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1878         dev_flow->verbs.attr.port = (uint8_t)priv->dev_port;
1879         return 0;
1880 }
1881
1882 /**
1883  * Remove the flow from the NIC but keeps it in memory.
1884  *
1885  * @param[in] dev
1886  *   Pointer to the Ethernet device structure.
1887  * @param[in, out] flow
1888  *   Pointer to flow structure.
1889  */
1890 static void
1891 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1892 {
1893         struct mlx5_priv *priv = dev->data->dev_private;
1894         struct mlx5_flow_handle *handle;
1895         uint32_t handle_idx;
1896
1897         if (!flow)
1898                 return;
1899         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1900                        handle_idx, handle, next) {
1901                 if (handle->drv_flow) {
1902                         claim_zero(mlx5_glue->destroy_flow(handle->drv_flow));
1903                         handle->drv_flow = NULL;
1904                 }
1905                 /* hrxq is union, don't touch it only the flag is set. */
1906                 if (handle->rix_hrxq &&
1907                     handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1908                         mlx5_hrxq_release(dev, handle->rix_hrxq);
1909                         handle->rix_hrxq = 0;
1910                 }
1911                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1912                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1913         }
1914 }
1915
1916 /**
1917  * Remove the flow from the NIC and the memory.
1918  *
1919  * @param[in] dev
1920  *   Pointer to the Ethernet device structure.
1921  * @param[in, out] flow
1922  *   Pointer to flow structure.
1923  */
1924 static void
1925 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1926 {
1927         struct mlx5_priv *priv = dev->data->dev_private;
1928         struct mlx5_flow_handle *handle;
1929
1930         if (!flow)
1931                 return;
1932         flow_verbs_remove(dev, flow);
1933         while (flow->dev_handles) {
1934                 uint32_t tmp_idx = flow->dev_handles;
1935
1936                 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1937                                    tmp_idx);
1938                 if (!handle)
1939                         return;
1940                 flow->dev_handles = handle->next.next;
1941                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1942                            tmp_idx);
1943         }
1944         if (flow->counter) {
1945                 flow_verbs_counter_release(dev, flow->counter);
1946                 flow->counter = 0;
1947         }
1948 }
1949
1950 /**
1951  * Apply the flow to the NIC.
1952  *
1953  * @param[in] dev
1954  *   Pointer to the Ethernet device structure.
1955  * @param[in, out] flow
1956  *   Pointer to flow structure.
1957  * @param[out] error
1958  *   Pointer to error structure.
1959  *
1960  * @return
1961  *   0 on success, a negative errno value otherwise and rte_errno is set.
1962  */
1963 static int
1964 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1965                  struct rte_flow_error *error)
1966 {
1967         struct mlx5_priv *priv = dev->data->dev_private;
1968         struct mlx5_flow_handle *handle;
1969         struct mlx5_flow *dev_flow;
1970         struct mlx5_hrxq *hrxq;
1971         uint32_t dev_handles;
1972         int err;
1973         int idx;
1974         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1975
1976         MLX5_ASSERT(wks);
1977         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
1978                 dev_flow = &wks->flows[idx];
1979                 handle = dev_flow->handle;
1980                 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1981                         MLX5_ASSERT(priv->drop_queue.hrxq);
1982                         hrxq = priv->drop_queue.hrxq;
1983                 } else {
1984                         uint32_t hrxq_idx;
1985                         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
1986
1987                         MLX5_ASSERT(rss_desc->queue_num);
1988                         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
1989                         rss_desc->hash_fields = dev_flow->hash_fields;
1990                         rss_desc->tunnel = !!(handle->layers &
1991                                               MLX5_FLOW_LAYER_TUNNEL);
1992                         rss_desc->shared_rss = 0;
1993                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
1994                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1995                                               hrxq_idx);
1996                         if (!hrxq) {
1997                                 rte_flow_error_set
1998                                         (error, rte_errno,
1999                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2000                                          "cannot get hash queue");
2001                                 goto error;
2002                         }
2003                         handle->rix_hrxq = hrxq_idx;
2004                 }
2005                 MLX5_ASSERT(hrxq);
2006                 handle->drv_flow = mlx5_glue->create_flow
2007                                         (hrxq->qp, &dev_flow->verbs.attr);
2008                 if (!handle->drv_flow) {
2009                         rte_flow_error_set(error, errno,
2010                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2011                                            NULL,
2012                                            "hardware refuses to create flow");
2013                         goto error;
2014                 }
2015                 if (priv->vmwa_context &&
2016                     handle->vf_vlan.tag && !handle->vf_vlan.created) {
2017                         /*
2018                          * The rule contains the VLAN pattern.
2019                          * For VF we are going to create VLAN
2020                          * interface to make hypervisor set correct
2021                          * e-Switch vport context.
2022                          */
2023                         mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
2024                 }
2025         }
2026         return 0;
2027 error:
2028         err = rte_errno; /* Save rte_errno before cleanup. */
2029         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2030                        dev_handles, handle, next) {
2031                 /* hrxq is union, don't touch it only the flag is set. */
2032                 if (handle->rix_hrxq &&
2033                     handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
2034                         mlx5_hrxq_release(dev, handle->rix_hrxq);
2035                         handle->rix_hrxq = 0;
2036                 }
2037                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
2038                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
2039         }
2040         rte_errno = err; /* Restore rte_errno. */
2041         return -rte_errno;
2042 }
2043
2044 /**
2045  * Query a flow.
2046  *
2047  * @see rte_flow_query()
2048  * @see rte_flow_ops
2049  */
2050 static int
2051 flow_verbs_query(struct rte_eth_dev *dev,
2052                  struct rte_flow *flow,
2053                  const struct rte_flow_action *actions,
2054                  void *data,
2055                  struct rte_flow_error *error)
2056 {
2057         int ret = -EINVAL;
2058
2059         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2060                 switch (actions->type) {
2061                 case RTE_FLOW_ACTION_TYPE_VOID:
2062                         break;
2063                 case RTE_FLOW_ACTION_TYPE_COUNT:
2064                         ret = flow_verbs_counter_query(dev, flow, data, error);
2065                         break;
2066                 default:
2067                         return rte_flow_error_set(error, ENOTSUP,
2068                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2069                                                   actions,
2070                                                   "action not supported");
2071                 }
2072         }
2073         return ret;
2074 }
2075
2076 static int
2077 flow_verbs_sync_domain(struct rte_eth_dev *dev, uint32_t domains,
2078                        uint32_t flags)
2079 {
2080         RTE_SET_USED(dev);
2081         RTE_SET_USED(domains);
2082         RTE_SET_USED(flags);
2083
2084         return 0;
2085 }
2086
2087 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
2088         .validate = flow_verbs_validate,
2089         .prepare = flow_verbs_prepare,
2090         .translate = flow_verbs_translate,
2091         .apply = flow_verbs_apply,
2092         .remove = flow_verbs_remove,
2093         .destroy = flow_verbs_destroy,
2094         .query = flow_verbs_query,
2095         .sync_domain = flow_verbs_sync_domain,
2096 };