drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_ip.h>
  18
  19 #include <mlx5_glue.h>
  20 #include <mlx5_prm.h>
  21 #include <mlx5_malloc.h>
  22
  23 #include "mlx5_defs.h"
  24 #include "mlx5.h"
  25 #include "mlx5_flow.h"
  26 #include "mlx5_rxtx.h"
  27
  28 #define VERBS_SPEC_INNER(item_flags) \
  29         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  30
  31 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
  32 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
  33         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
  34 };
  35
  36 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
  37 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
  38         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
  39         { 9, 10, 11 }, { 12, 13, 14 },
  40 };
  41
  42 /**
  43  * Discover the maximum number of priority available.
  44  *
  45  * @param[in] dev
  46  *   Pointer to the Ethernet device structure.
  47  *
  48  * @return
  49  *   number of supported flow priority on success, a negative errno
  50  *   value otherwise and rte_errno is set.
  51  */
  52 int
  53 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
  54 {
  55         struct mlx5_priv *priv = dev->data->dev_private;
  56         struct {
  57                 struct ibv_flow_attr attr;
  58                 struct ibv_flow_spec_eth eth;
  59                 struct ibv_flow_spec_action_drop drop;
  60         } flow_attr = {
  61                 .attr = {
  62                         .num_of_specs = 2,
  63                         .port = (uint8_t)priv->dev_port,
  64                 },
  65                 .eth = {
  66                         .type = IBV_FLOW_SPEC_ETH,
  67                         .size = sizeof(struct ibv_flow_spec_eth),
  68                 },
  69                 .drop = {
  70                         .size = sizeof(struct ibv_flow_spec_action_drop),
  71                         .type = IBV_FLOW_SPEC_ACTION_DROP,
  72                 },
  73         };
  74         struct ibv_flow *flow;
  75         struct mlx5_hrxq *drop = priv->drop_queue.hrxq;
  76         uint16_t vprio[] = { 8, 16 };
  77         int i;
  78         int priority = 0;
  79
  80         if (!drop->qp) {
  81                 rte_errno = ENOTSUP;
  82                 return -rte_errno;
  83         }
  84         for (i = 0; i != RTE_DIM(vprio); i++) {
  85                 flow_attr.attr.priority = vprio[i] - 1;
  86                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
  87                 if (!flow)
  88                         break;
  89                 claim_zero(mlx5_glue->destroy_flow(flow));
  90                 priority = vprio[i];
  91         }
  92         switch (priority) {
  93         case 8:
  94                 priority = RTE_DIM(priority_map_3);
  95                 break;
  96         case 16:
  97                 priority = RTE_DIM(priority_map_5);
  98                 break;
  99         default:
 100                 rte_errno = ENOTSUP;
 101                 DRV_LOG(ERR,
 102                         "port %u verbs maximum priority: %d expected 8/16",
 103                         dev->data->port_id, priority);
 104                 return -rte_errno;
 105         }
 106         DRV_LOG(INFO, "port %u flow maximum priority: %d",
 107                 dev->data->port_id, priority);
 108         return priority;
 109 }
 110
 111 /**
 112  * Adjust flow priority based on the highest layer and the request priority.
 113  *
 114  * @param[in] dev
 115  *   Pointer to the Ethernet device structure.
 116  * @param[in] priority
 117  *   The rule base priority.
 118  * @param[in] subpriority
 119  *   The priority based on the items.
 120  *
 121  * @return
 122  *   The new priority.
 123  */
 124 uint32_t
 125 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 126                                    uint32_t subpriority)
 127 {
 128         uint32_t res = 0;
 129         struct mlx5_priv *priv = dev->data->dev_private;
 130
 131         switch (priv->config.flow_prio) {
 132         case RTE_DIM(priority_map_3):
 133                 res = priority_map_3[priority][subpriority];
 134                 break;
 135         case RTE_DIM(priority_map_5):
 136                 res = priority_map_5[priority][subpriority];
 137                 break;
 138         }
 139         return  res;
 140 }
 141
 142 /**
 143  * Get Verbs flow counter by index.
 144  *
 145  * @param[in] dev
 146  *   Pointer to the Ethernet device structure.
 147  * @param[in] idx
 148  *   mlx5 flow counter index in the container.
 149  * @param[out] ppool
 150  *   mlx5 flow counter pool in the container,
 151  *
 152  * @return
 153  *   A pointer to the counter, NULL otherwise.
 154  */
 155 static struct mlx5_flow_counter *
 156 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
 157                               uint32_t idx,
 158                               struct mlx5_flow_counter_pool **ppool)
 159 {
 160         struct mlx5_priv *priv = dev->data->dev_private;
 161         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 162         struct mlx5_flow_counter_pool *pool;
 163
 164         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
 165         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
 166         MLX5_ASSERT(pool);
 167         if (ppool)
 168                 *ppool = pool;
 169         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
 170 }
 171
 172 /**
 173  * Create Verbs flow counter with Verbs library.
 174  *
 175  * @param[in] dev
 176  *   Pointer to the Ethernet device structure.
 177  * @param[in, out] counter
 178  *   mlx5 flow counter object, contains the counter id,
 179  *   handle of created Verbs flow counter is returned
 180  *   in cs field (if counters are supported).
 181  *
 182  * @return
 183  *   0 On success else a negative errno value is returned
 184  *   and rte_errno is set.
 185  */
 186 static int
 187 flow_verbs_counter_create(struct rte_eth_dev *dev,
 188                           struct mlx5_flow_counter *counter)
 189 {
 190 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 191         struct mlx5_priv *priv = dev->data->dev_private;
 192         struct ibv_context *ctx = priv->sh->ctx;
 193         struct ibv_counter_set_init_attr init = {
 194                          .counter_set_id = counter->shared_info.id};
 195
 196         counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init);
 197         if (!counter->dcs_when_free) {
 198                 rte_errno = ENOTSUP;
 199                 return -ENOTSUP;
 200         }
 201         return 0;
 202 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 203         struct mlx5_priv *priv = dev->data->dev_private;
 204         struct ibv_context *ctx = priv->sh->ctx;
 205         struct ibv_counters_init_attr init = {0};
 206         struct ibv_counter_attach_attr attach;
 207         int ret;
 208
 209         memset(&attach, 0, sizeof(attach));
 210         counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init);
 211         if (!counter->dcs_when_free) {
 212                 rte_errno = ENOTSUP;
 213                 return -ENOTSUP;
 214         }
 215         attach.counter_desc = IBV_COUNTER_PACKETS;
 216         attach.index = 0;
 217         ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL);
 218         if (!ret) {
 219                 attach.counter_desc = IBV_COUNTER_BYTES;
 220                 attach.index = 1;
 221                 ret = mlx5_glue->attach_counters
 222                                         (counter->dcs_when_free, &attach, NULL);
 223         }
 224         if (ret) {
 225                 claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free));
 226                 counter->dcs_when_free = NULL;
 227                 rte_errno = ret;
 228                 return -ret;
 229         }
 230         return 0;
 231 #else
 232         (void)dev;
 233         (void)counter;
 234         rte_errno = ENOTSUP;
 235         return -ENOTSUP;
 236 #endif
 237 }
 238
 239 /**
 240  * Get a flow counter.
 241  *
 242  * @param[in] dev
 243  *   Pointer to the Ethernet device structure.
 244  * @param[in] shared
 245  *   Indicate if this counter is shared with other flows.
 246  * @param[in] id
 247  *   Counter identifier.
 248  *
 249  * @return
 250  *   Index to the counter, 0 otherwise and rte_errno is set.
 251  */
 252 static uint32_t
 253 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 254 {
 255         struct mlx5_priv *priv = dev->data->dev_private;
 256         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 257         struct mlx5_flow_counter_pool *pool = NULL;
 258         struct mlx5_flow_counter *cnt = NULL;
 259         union mlx5_l3t_data data;
 260         uint32_t n_valid = cmng->n_valid;
 261         uint32_t pool_idx, cnt_idx;
 262         uint32_t i;
 263         int ret;
 264
 265         if (shared && !mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) &&
 266             data.dword)
 267                 return data.dword;
 268         for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 269                 pool = cmng->pools[pool_idx];
 270                 if (!pool)
 271                         continue;
 272                 cnt = TAILQ_FIRST(&pool->counters[0]);
 273                 if (cnt)
 274                         break;
 275         }
 276         if (!cnt) {
 277                 struct mlx5_flow_counter_pool **pools;
 278                 uint32_t size;
 279
 280                 if (n_valid == cmng->n) {
 281                         /* Resize the container pool array. */
 282                         size = sizeof(struct mlx5_flow_counter_pool *) *
 283                                      (n_valid + MLX5_CNT_CONTAINER_RESIZE);
 284                         pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0,
 285                                             SOCKET_ID_ANY);
 286                         if (!pools)
 287                                 return 0;
 288                         if (n_valid) {
 289                                 memcpy(pools, cmng->pools,
 290                                        sizeof(struct mlx5_flow_counter_pool *) *
 291                                        n_valid);
 292                                 mlx5_free(cmng->pools);
 293                         }
 294                         cmng->pools = pools;
 295                         cmng->n += MLX5_CNT_CONTAINER_RESIZE;
 296                 }
 297                 /* Allocate memory for new pool*/
 298                 size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL;
 299                 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
 300                 if (!pool)
 301                         return 0;
 302                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
 303                         cnt = MLX5_POOL_GET_CNT(pool, i);
 304                         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 305                 }
 306                 cnt = MLX5_POOL_GET_CNT(pool, 0);
 307                 cmng->pools[n_valid] = pool;
 308                 pool_idx = n_valid;
 309                 cmng->n_valid++;
 310         }
 311         TAILQ_REMOVE(&pool->counters[0], cnt, next);
 312         i = MLX5_CNT_ARRAY_IDX(pool, cnt);
 313         cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i);
 314         if (shared) {
 315                 data.dword = cnt_idx;
 316                 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
 317                         return 0;
 318                 cnt->shared_info.id = id;
 319                 cnt_idx |= MLX5_CNT_SHARED_OFFSET;
 320         }
 321         /* Create counter with Verbs. */
 322         ret = flow_verbs_counter_create(dev, cnt);
 323         if (!ret) {
 324                 cnt->dcs_when_active = cnt->dcs_when_free;
 325                 cnt->hits = 0;
 326                 cnt->bytes = 0;
 327                 return cnt_idx;
 328         }
 329         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 330         /* Some error occurred in Verbs library. */
 331         rte_errno = -ret;
 332         return 0;
 333 }
 334
 335 /**
 336  * Release a flow counter.
 337  *
 338  * @param[in] dev
 339  *   Pointer to the Ethernet device structure.
 340  * @param[in] counter
 341  *   Index to the counter handler.
 342  */
 343 static void
 344 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
 345 {
 346         struct mlx5_priv *priv = dev->data->dev_private;
 347         struct mlx5_flow_counter_pool *pool;
 348         struct mlx5_flow_counter *cnt;
 349
 350         cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool);
 351         if (IS_SHARED_CNT(counter) &&
 352             mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
 353                 return;
 354 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 355         claim_zero(mlx5_glue->destroy_counter_set
 356                         ((struct ibv_counter_set *)cnt->dcs_when_active));
 357 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 358         claim_zero(mlx5_glue->destroy_counters
 359                                 ((struct ibv_counters *)cnt->dcs_when_active));
 360 #endif
 361         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 362 }
 363
 364 /**
 365  * Query a flow counter via Verbs library call.
 366  *
 367  * @see rte_flow_query()
 368  * @see rte_flow_ops
 369  */
 370 static int
 371 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 372                          struct rte_flow *flow, void *data,
 373                          struct rte_flow_error *error)
 374 {
 375 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 376         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 377         if (flow->counter) {
 378                 struct mlx5_flow_counter_pool *pool;
 379                 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
 380                                                 (dev, flow->counter, &pool);
 381                 struct rte_flow_query_count *qc = data;
 382                 uint64_t counters[2] = {0, 0};
 383 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 384                 struct ibv_query_counter_set_attr query_cs_attr = {
 385                         .dcs_when_free = (struct ibv_counter_set *)
 386                                                 cnt->dcs_when_active,
 387                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 388                 };
 389                 struct ibv_counter_set_data query_out = {
 390                         .out = counters,
 391                         .outlen = 2 * sizeof(uint64_t),
 392                 };
 393                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 394                                                        &query_out);
 395 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 396                 int err = mlx5_glue->query_counters
 397                         ((struct ibv_counters *)cnt->dcs_when_active, counters,
 398                                 RTE_DIM(counters),
 399                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 400 #endif
 401                 if (err)
 402                         return rte_flow_error_set
 403                                 (error, err,
 404                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 405                                  NULL,
 406                                  "cannot read counter");
 407                 qc->hits_set = 1;
 408                 qc->bytes_set = 1;
 409                 qc->hits = counters[0] - cnt->hits;
 410                 qc->bytes = counters[1] - cnt->bytes;
 411                 if (qc->reset) {
 412                         cnt->hits = counters[0];
 413                         cnt->bytes = counters[1];
 414                 }
 415                 return 0;
 416         }
 417         return rte_flow_error_set(error, EINVAL,
 418                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 419                                   NULL,
 420                                   "flow does not have counter");
 421 #else
 422         (void)flow;
 423         (void)data;
 424         return rte_flow_error_set(error, ENOTSUP,
 425                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 426                                   NULL,
 427                                   "counters are not available");
 428 #endif
 429 }
 430
 431 /**
 432  * Add a verbs item specification into @p verbs.
 433  *
 434  * @param[out] verbs
 435  *   Pointer to verbs structure.
 436  * @param[in] src
 437  *   Create specification.
 438  * @param[in] size
 439  *   Size in bytes of the specification to copy.
 440  */
 441 static void
 442 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
 443                     void *src, unsigned int size)
 444 {
 445         void *dst;
 446
 447         if (!verbs)
 448                 return;
 449         MLX5_ASSERT(verbs->specs);
 450         dst = (void *)(verbs->specs + verbs->size);
 451         memcpy(dst, src, size);
 452         ++verbs->attr.num_of_specs;
 453         verbs->size += size;
 454 }
 455
 456 /**
 457  * Convert the @p item into a Verbs specification. This function assumes that
 458  * the input is valid and that there is space to insert the requested item
 459  * into the flow.
 460  *
 461  * @param[in, out] dev_flow
 462  *   Pointer to dev_flow structure.
 463  * @param[in] item
 464  *   Item specification.
 465  * @param[in] item_flags
 466  *   Parsed item flags.
 467  */
 468 static void
 469 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 470                               const struct rte_flow_item *item,
 471                               uint64_t item_flags)
 472 {
 473         const struct rte_flow_item_eth *spec = item->spec;
 474         const struct rte_flow_item_eth *mask = item->mask;
 475         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 476         struct ibv_flow_spec_eth eth = {
 477                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 478                 .size = size,
 479         };
 480
 481         if (!mask)
 482                 mask = &rte_flow_item_eth_mask;
 483         if (spec) {
 484                 unsigned int i;
 485
 486                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 487                         RTE_ETHER_ADDR_LEN);
 488                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 489                         RTE_ETHER_ADDR_LEN);
 490                 eth.val.ether_type = spec->type;
 491                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 492                         RTE_ETHER_ADDR_LEN);
 493                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 494                         RTE_ETHER_ADDR_LEN);
 495                 eth.mask.ether_type = mask->type;
 496                 /* Remove unwanted bits from values. */
 497                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 498                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 499                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 500                 }
 501                 eth.val.ether_type &= eth.mask.ether_type;
 502         }
 503         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 504 }
 505
 506 /**
 507  * Update the VLAN tag in the Verbs Ethernet specification.
 508  * This function assumes that the input is valid and there is space to add
 509  * the requested item.
 510  *
 511  * @param[in, out] attr
 512  *   Pointer to Verbs attributes structure.
 513  * @param[in] eth
 514  *   Verbs structure containing the VLAN information to copy.
 515  */
 516 static void
 517 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 518                             struct ibv_flow_spec_eth *eth)
 519 {
 520         unsigned int i;
 521         const enum ibv_flow_spec_type search = eth->type;
 522         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 523                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 524
 525         for (i = 0; i != attr->num_of_specs; ++i) {
 526                 if (hdr->type == search) {
 527                         struct ibv_flow_spec_eth *e =
 528                                 (struct ibv_flow_spec_eth *)hdr;
 529
 530                         e->val.vlan_tag = eth->val.vlan_tag;
 531                         e->mask.vlan_tag = eth->mask.vlan_tag;
 532                         e->val.ether_type = eth->val.ether_type;
 533                         e->mask.ether_type = eth->mask.ether_type;
 534                         break;
 535                 }
 536                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 537         }
 538 }
 539
 540 /**
 541  * Convert the @p item into a Verbs specification. This function assumes that
 542  * the input is valid and that there is space to insert the requested item
 543  * into the flow.
 544  *
 545  * @param[in, out] dev_flow
 546  *   Pointer to dev_flow structure.
 547  * @param[in] item
 548  *   Item specification.
 549  * @param[in] item_flags
 550  *   Parsed item flags.
 551  */
 552 static void
 553 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 554                                const struct rte_flow_item *item,
 555                                uint64_t item_flags)
 556 {
 557         const struct rte_flow_item_vlan *spec = item->spec;
 558         const struct rte_flow_item_vlan *mask = item->mask;
 559         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 560         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 561         struct ibv_flow_spec_eth eth = {
 562                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 563                 .size = size,
 564         };
 565         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 566                                       MLX5_FLOW_LAYER_OUTER_L2;
 567
 568         if (!mask)
 569                 mask = &rte_flow_item_vlan_mask;
 570         if (spec) {
 571                 eth.val.vlan_tag = spec->tci;
 572                 eth.mask.vlan_tag = mask->tci;
 573                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 574                 eth.val.ether_type = spec->inner_type;
 575                 eth.mask.ether_type = mask->inner_type;
 576                 eth.val.ether_type &= eth.mask.ether_type;
 577         }
 578         if (!(item_flags & l2m))
 579                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 580         else
 581                 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, &eth);
 582         if (!tunnel)
 583                 dev_flow->handle->vf_vlan.tag =
 584                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 585 }
 586
 587 /**
 588  * Convert the @p item into a Verbs specification. This function assumes that
 589  * the input is valid and that there is space to insert the requested item
 590  * into the flow.
 591  *
 592  * @param[in, out] dev_flow
 593  *   Pointer to dev_flow structure.
 594  * @param[in] item
 595  *   Item specification.
 596  * @param[in] item_flags
 597  *   Parsed item flags.
 598  */
 599 static void
 600 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 601                                const struct rte_flow_item *item,
 602                                uint64_t item_flags)
 603 {
 604         const struct rte_flow_item_ipv4 *spec = item->spec;
 605         const struct rte_flow_item_ipv4 *mask = item->mask;
 606         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 607         struct ibv_flow_spec_ipv4_ext ipv4 = {
 608                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 609                 .size = size,
 610         };
 611
 612         if (!mask)
 613                 mask = &rte_flow_item_ipv4_mask;
 614         if (spec) {
 615                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 616                         .src_ip = spec->hdr.src_addr,
 617                         .dst_ip = spec->hdr.dst_addr,
 618                         .proto = spec->hdr.next_proto_id,
 619                         .tos = spec->hdr.type_of_service,
 620                 };
 621                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 622                         .src_ip = mask->hdr.src_addr,
 623                         .dst_ip = mask->hdr.dst_addr,
 624                         .proto = mask->hdr.next_proto_id,
 625                         .tos = mask->hdr.type_of_service,
 626                 };
 627                 /* Remove unwanted bits from values. */
 628                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 629                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 630                 ipv4.val.proto &= ipv4.mask.proto;
 631                 ipv4.val.tos &= ipv4.mask.tos;
 632         }
 633         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 634 }
 635
 636 /**
 637  * Convert the @p item into a Verbs specification. This function assumes that
 638  * the input is valid and that there is space to insert the requested item
 639  * into the flow.
 640  *
 641  * @param[in, out] dev_flow
 642  *   Pointer to dev_flow structure.
 643  * @param[in] item
 644  *   Item specification.
 645  * @param[in] item_flags
 646  *   Parsed item flags.
 647  */
 648 static void
 649 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 650                                const struct rte_flow_item *item,
 651                                uint64_t item_flags)
 652 {
 653         const struct rte_flow_item_ipv6 *spec = item->spec;
 654         const struct rte_flow_item_ipv6 *mask = item->mask;
 655         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 656         struct ibv_flow_spec_ipv6 ipv6 = {
 657                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 658                 .size = size,
 659         };
 660
 661         if (!mask)
 662                 mask = &rte_flow_item_ipv6_mask;
 663         if (spec) {
 664                 unsigned int i;
 665                 uint32_t vtc_flow_val;
 666                 uint32_t vtc_flow_mask;
 667
 668                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 669                        RTE_DIM(ipv6.val.src_ip));
 670                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 671                        RTE_DIM(ipv6.val.dst_ip));
 672                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 673                        RTE_DIM(ipv6.mask.src_ip));
 674                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 675                        RTE_DIM(ipv6.mask.dst_ip));
 676                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 677                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 678                 ipv6.val.flow_label =
 679                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 680                                          RTE_IPV6_HDR_FL_SHIFT);
 681                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 682                                          RTE_IPV6_HDR_TC_SHIFT;
 683                 ipv6.val.next_hdr = spec->hdr.proto;
 684                 ipv6.mask.flow_label =
 685                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 686                                          RTE_IPV6_HDR_FL_SHIFT);
 687                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 688                                           RTE_IPV6_HDR_TC_SHIFT;
 689                 ipv6.mask.next_hdr = mask->hdr.proto;
 690                 /* Remove unwanted bits from values. */
 691                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 692                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 693                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 694                 }
 695                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 696                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 697                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 698         }
 699         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 700 }
 701
 702 /**
 703  * Convert the @p item into a Verbs specification. This function assumes that
 704  * the input is valid and that there is space to insert the requested item
 705  * into the flow.
 706  *
 707  * @param[in, out] dev_flow
 708  *   Pointer to dev_flow structure.
 709  * @param[in] item
 710  *   Item specification.
 711  * @param[in] item_flags
 712  *   Parsed item flags.
 713  */
 714 static void
 715 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 716                               const struct rte_flow_item *item,
 717                               uint64_t item_flags __rte_unused)
 718 {
 719         const struct rte_flow_item_tcp *spec = item->spec;
 720         const struct rte_flow_item_tcp *mask = item->mask;
 721         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 722         struct ibv_flow_spec_tcp_udp tcp = {
 723                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 724                 .size = size,
 725         };
 726
 727         if (!mask)
 728                 mask = &rte_flow_item_tcp_mask;
 729         if (spec) {
 730                 tcp.val.dst_port = spec->hdr.dst_port;
 731                 tcp.val.src_port = spec->hdr.src_port;
 732                 tcp.mask.dst_port = mask->hdr.dst_port;
 733                 tcp.mask.src_port = mask->hdr.src_port;
 734                 /* Remove unwanted bits from values. */
 735                 tcp.val.src_port &= tcp.mask.src_port;
 736                 tcp.val.dst_port &= tcp.mask.dst_port;
 737         }
 738         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 739 }
 740
 741 /**
 742  * Convert the @p item into a Verbs specification. This function assumes that
 743  * the input is valid and that there is space to insert the requested item
 744  * into the flow.
 745  *
 746  * @param[in, out] dev_flow
 747  *   Pointer to dev_flow structure.
 748  * @param[in] item
 749  *   Item specification.
 750  * @param[in] item_flags
 751  *   Parsed item flags.
 752  */
 753 static void
 754 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 755                               const struct rte_flow_item *item,
 756                               uint64_t item_flags __rte_unused)
 757 {
 758         const struct rte_flow_item_udp *spec = item->spec;
 759         const struct rte_flow_item_udp *mask = item->mask;
 760         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 761         struct ibv_flow_spec_tcp_udp udp = {
 762                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 763                 .size = size,
 764         };
 765
 766         if (!mask)
 767                 mask = &rte_flow_item_udp_mask;
 768         if (spec) {
 769                 udp.val.dst_port = spec->hdr.dst_port;
 770                 udp.val.src_port = spec->hdr.src_port;
 771                 udp.mask.dst_port = mask->hdr.dst_port;
 772                 udp.mask.src_port = mask->hdr.src_port;
 773                 /* Remove unwanted bits from values. */
 774                 udp.val.src_port &= udp.mask.src_port;
 775                 udp.val.dst_port &= udp.mask.dst_port;
 776         }
 777         item++;
 778         while (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 779                 item++;
 780         if (!(udp.val.dst_port & udp.mask.dst_port)) {
 781                 switch ((item)->type) {
 782                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 783                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN);
 784                         udp.mask.dst_port = 0xffff;
 785                         break;
 786                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 787                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE);
 788                         udp.mask.dst_port = 0xffff;
 789                         break;
 790                 case RTE_FLOW_ITEM_TYPE_MPLS:
 791                         udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS);
 792                         udp.mask.dst_port = 0xffff;
 793                         break;
 794                 default:
 795                         break;
 796                 }
 797         }
 798
 799         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 800 }
 801
 802 /**
 803  * Convert the @p item into a Verbs specification. This function assumes that
 804  * the input is valid and that there is space to insert the requested item
 805  * into the flow.
 806  *
 807  * @param[in, out] dev_flow
 808  *   Pointer to dev_flow structure.
 809  * @param[in] item
 810  *   Item specification.
 811  * @param[in] item_flags
 812  *   Parsed item flags.
 813  */
 814 static void
 815 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 816                                 const struct rte_flow_item *item,
 817                                 uint64_t item_flags __rte_unused)
 818 {
 819         const struct rte_flow_item_vxlan *spec = item->spec;
 820         const struct rte_flow_item_vxlan *mask = item->mask;
 821         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 822         struct ibv_flow_spec_tunnel vxlan = {
 823                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 824                 .size = size,
 825         };
 826         union vni {
 827                 uint32_t vlan_id;
 828                 uint8_t vni[4];
 829         } id = { .vlan_id = 0, };
 830
 831         if (!mask)
 832                 mask = &rte_flow_item_vxlan_mask;
 833         if (spec) {
 834                 memcpy(&id.vni[1], spec->vni, 3);
 835                 vxlan.val.tunnel_id = id.vlan_id;
 836                 memcpy(&id.vni[1], mask->vni, 3);
 837                 vxlan.mask.tunnel_id = id.vlan_id;
 838                 /* Remove unwanted bits from values. */
 839                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 840         }
 841         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 842 }
 843
 844 /**
 845  * Convert the @p item into a Verbs specification. This function assumes that
 846  * the input is valid and that there is space to insert the requested item
 847  * into the flow.
 848  *
 849  * @param[in, out] dev_flow
 850  *   Pointer to dev_flow structure.
 851  * @param[in] item
 852  *   Item specification.
 853  * @param[in] item_flags
 854  *   Parsed item flags.
 855  */
 856 static void
 857 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 858                                     const struct rte_flow_item *item,
 859                                     uint64_t item_flags __rte_unused)
 860 {
 861         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 862         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 863         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 864         struct ibv_flow_spec_tunnel vxlan_gpe = {
 865                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 866                 .size = size,
 867         };
 868         union vni {
 869                 uint32_t vlan_id;
 870                 uint8_t vni[4];
 871         } id = { .vlan_id = 0, };
 872
 873         if (!mask)
 874                 mask = &rte_flow_item_vxlan_gpe_mask;
 875         if (spec) {
 876                 memcpy(&id.vni[1], spec->vni, 3);
 877                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 878                 memcpy(&id.vni[1], mask->vni, 3);
 879                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 880                 /* Remove unwanted bits from values. */
 881                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 882         }
 883         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 884 }
 885
 886 /**
 887  * Update the protocol in Verbs IPv4/IPv6 spec.
 888  *
 889  * @param[in, out] attr
 890  *   Pointer to Verbs attributes structure.
 891  * @param[in] search
 892  *   Specification type to search in order to update the IP protocol.
 893  * @param[in] protocol
 894  *   Protocol value to set if none is present in the specification.
 895  */
 896 static void
 897 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 898                                        enum ibv_flow_spec_type search,
 899                                        uint8_t protocol)
 900 {
 901         unsigned int i;
 902         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 903                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 904
 905         if (!attr)
 906                 return;
 907         for (i = 0; i != attr->num_of_specs; ++i) {
 908                 if (hdr->type == search) {
 909                         union {
 910                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 911                                 struct ibv_flow_spec_ipv6 *ipv6;
 912                         } ip;
 913
 914                         switch (search) {
 915                         case IBV_FLOW_SPEC_IPV4_EXT:
 916                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 917                                 if (!ip.ipv4->val.proto) {
 918                                         ip.ipv4->val.proto = protocol;
 919                                         ip.ipv4->mask.proto = 0xff;
 920                                 }
 921                                 break;
 922                         case IBV_FLOW_SPEC_IPV6:
 923                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 924                                 if (!ip.ipv6->val.next_hdr) {
 925                                         ip.ipv6->val.next_hdr = protocol;
 926                                         ip.ipv6->mask.next_hdr = 0xff;
 927                                 }
 928                                 break;
 929                         default:
 930                                 break;
 931                         }
 932                         break;
 933                 }
 934                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 935         }
 936 }
 937
 938 /**
 939  * Convert the @p item into a Verbs specification. This function assumes that
 940  * the input is valid and that there is space to insert the requested item
 941  * into the flow.
 942  *
 943  * @param[in, out] dev_flow
 944  *   Pointer to dev_flow structure.
 945  * @param[in] item
 946  *   Item specification.
 947  * @param[in] item_flags
 948  *   Parsed item flags.
 949  */
 950 static void
 951 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 952                               const struct rte_flow_item *item __rte_unused,
 953                               uint64_t item_flags)
 954 {
 955         struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
 956 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 957         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 958         struct ibv_flow_spec_tunnel tunnel = {
 959                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 960                 .size = size,
 961         };
 962 #else
 963         const struct rte_flow_item_gre *spec = item->spec;
 964         const struct rte_flow_item_gre *mask = item->mask;
 965         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 966         struct ibv_flow_spec_gre tunnel = {
 967                 .type = IBV_FLOW_SPEC_GRE,
 968                 .size = size,
 969         };
 970
 971         if (!mask)
 972                 mask = &rte_flow_item_gre_mask;
 973         if (spec) {
 974                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 975                 tunnel.val.protocol = spec->protocol;
 976                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 977                 tunnel.mask.protocol = mask->protocol;
 978                 /* Remove unwanted bits from values. */
 979                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 980                 tunnel.val.protocol &= tunnel.mask.protocol;
 981                 tunnel.val.key &= tunnel.mask.key;
 982         }
 983 #endif
 984         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 985                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 986                                                        IBV_FLOW_SPEC_IPV4_EXT,
 987                                                        IPPROTO_GRE);
 988         else
 989                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 990                                                        IBV_FLOW_SPEC_IPV6,
 991                                                        IPPROTO_GRE);
 992         flow_verbs_spec_add(verbs, &tunnel, size);
 993 }
 994
 995 /**
 996  * Convert the @p action into a Verbs specification. This function assumes that
 997  * the input is valid and that there is space to insert the requested action
 998  * into the flow. This function also return the action that was added.
 999  *
1000  * @param[in, out] dev_flow
1001  *   Pointer to dev_flow structure.
1002  * @param[in] item
1003  *   Item specification.
1004  * @param[in] item_flags
1005  *   Parsed item flags.
1006  */
1007 static void
1008 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
1009                                const struct rte_flow_item *item __rte_unused,
1010                                uint64_t item_flags __rte_unused)
1011 {
1012 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1013         const struct rte_flow_item_mpls *spec = item->spec;
1014         const struct rte_flow_item_mpls *mask = item->mask;
1015         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
1016         struct ibv_flow_spec_mpls mpls = {
1017                 .type = IBV_FLOW_SPEC_MPLS,
1018                 .size = size,
1019         };
1020
1021         if (!mask)
1022                 mask = &rte_flow_item_mpls_mask;
1023         if (spec) {
1024                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
1025                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
1026                 /* Remove unwanted bits from values.  */
1027                 mpls.val.label &= mpls.mask.label;
1028         }
1029         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
1030 #endif
1031 }
1032
1033 /**
1034  * Convert the @p action into a Verbs specification. This function assumes that
1035  * the input is valid and that there is space to insert the requested action
1036  * into the flow.
1037  *
1038  * @param[in] dev_flow
1039  *   Pointer to mlx5_flow.
1040  * @param[in] action
1041  *   Action configuration.
1042  */
1043 static void
1044 flow_verbs_translate_action_drop
1045         (struct mlx5_flow *dev_flow,
1046          const struct rte_flow_action *action __rte_unused)
1047 {
1048         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1049         struct ibv_flow_spec_action_drop drop = {
1050                         .type = IBV_FLOW_SPEC_ACTION_DROP,
1051                         .size = size,
1052         };
1053
1054         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
1055 }
1056
1057 /**
1058  * Convert the @p action into a Verbs specification. This function assumes that
1059  * the input is valid and that there is space to insert the requested action
1060  * into the flow.
1061  *
1062  * @param[in] rss_desc
1063  *   Pointer to mlx5_flow_rss_desc.
1064  * @param[in] action
1065  *   Action configuration.
1066  */
1067 static void
1068 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc,
1069                                   const struct rte_flow_action *action)
1070 {
1071         const struct rte_flow_action_queue *queue = action->conf;
1072
1073         rss_desc->queue[0] = queue->index;
1074         rss_desc->queue_num = 1;
1075 }
1076
1077 /**
1078  * Convert the @p action into a Verbs specification. This function assumes that
1079  * the input is valid and that there is space to insert the requested action
1080  * into the flow.
1081  *
1082  * @param[in] rss_desc
1083  *   Pointer to mlx5_flow_rss_desc.
1084  * @param[in] action
1085  *   Action configuration.
1086  */
1087 static void
1088 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc,
1089                                 const struct rte_flow_action *action)
1090 {
1091         const struct rte_flow_action_rss *rss = action->conf;
1092         const uint8_t *rss_key;
1093
1094         memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t));
1095         rss_desc->queue_num = rss->queue_num;
1096         /* NULL RSS key indicates default RSS key. */
1097         rss_key = !rss->key ? rss_hash_default_key : rss->key;
1098         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
1099         /*
1100          * rss->level and rss.types should be set in advance when expanding
1101          * items for RSS.
1102          */
1103 }
1104
1105 /**
1106  * Convert the @p action into a Verbs specification. This function assumes that
1107  * the input is valid and that there is space to insert the requested action
1108  * into the flow.
1109  *
1110  * @param[in] dev_flow
1111  *   Pointer to mlx5_flow.
1112  * @param[in] action
1113  *   Action configuration.
1114  */
1115 static void
1116 flow_verbs_translate_action_flag
1117         (struct mlx5_flow *dev_flow,
1118          const struct rte_flow_action *action __rte_unused)
1119 {
1120         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1121         struct ibv_flow_spec_action_tag tag = {
1122                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1123                 .size = size,
1124                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1125         };
1126
1127         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1128 }
1129
1130 /**
1131  * Convert the @p action into a Verbs specification. This function assumes that
1132  * the input is valid and that there is space to insert the requested action
1133  * into the flow.
1134  *
1135  * @param[in] dev_flow
1136  *   Pointer to mlx5_flow.
1137  * @param[in] action
1138  *   Action configuration.
1139  */
1140 static void
1141 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1142                                  const struct rte_flow_action *action)
1143 {
1144         const struct rte_flow_action_mark *mark = action->conf;
1145         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1146         struct ibv_flow_spec_action_tag tag = {
1147                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1148                 .size = size,
1149                 .tag_id = mlx5_flow_mark_set(mark->id),
1150         };
1151
1152         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1153 }
1154
1155 /**
1156  * Convert the @p action into a Verbs specification. This function assumes that
1157  * the input is valid and that there is space to insert the requested action
1158  * into the flow.
1159  *
1160  * @param[in] dev
1161  *   Pointer to the Ethernet device structure.
1162  * @param[in] action
1163  *   Action configuration.
1164  * @param[in] dev_flow
1165  *   Pointer to mlx5_flow.
1166  * @param[out] error
1167  *   Pointer to error structure.
1168  *
1169  * @return
1170  *   0 On success else a negative errno value is returned and rte_errno is set.
1171  */
1172 static int
1173 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1174                                   const struct rte_flow_action *action,
1175                                   struct rte_eth_dev *dev,
1176                                   struct rte_flow_error *error)
1177 {
1178         const struct rte_flow_action_count *count = action->conf;
1179         struct rte_flow *flow = dev_flow->flow;
1180 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1181         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1182         struct mlx5_flow_counter_pool *pool;
1183         struct mlx5_flow_counter *cnt = NULL;
1184         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1185         struct ibv_flow_spec_counter_action counter = {
1186                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1187                 .size = size,
1188         };
1189 #endif
1190
1191         if (!flow->counter) {
1192                 flow->counter = flow_verbs_counter_new(dev, count->shared,
1193                                                        count->id);
1194                 if (!flow->counter)
1195                         return rte_flow_error_set(error, rte_errno,
1196                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1197                                                   action,
1198                                                   "cannot get counter"
1199                                                   " context.");
1200         }
1201 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1202         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1203         counter.counter_set_handle =
1204                 ((struct ibv_counter_set *)cnt->dcs_when_active)->handle;
1205         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1206 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1207         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1208         counter.counters = (struct ibv_counters *)cnt->dcs_when_active;
1209         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1210 #endif
1211         return 0;
1212 }
1213
1214 /**
1215  * Internal validation function. For validating both actions and items.
1216  *
1217  * @param[in] dev
1218  *   Pointer to the Ethernet device structure.
1219  * @param[in] attr
1220  *   Pointer to the flow attributes.
1221  * @param[in] items
1222  *   Pointer to the list of items.
1223  * @param[in] actions
1224  *   Pointer to the list of actions.
1225  * @param[in] external
1226  *   This flow rule is created by request external to PMD.
1227  * @param[in] hairpin
1228  *   Number of hairpin TX actions, 0 means classic flow.
1229  * @param[out] error
1230  *   Pointer to the error structure.
1231  *
1232  * @return
1233  *   0 on success, a negative errno value otherwise and rte_errno is set.
1234  */
1235 static int
1236 flow_verbs_validate(struct rte_eth_dev *dev,
1237                     const struct rte_flow_attr *attr,
1238                     const struct rte_flow_item items[],
1239                     const struct rte_flow_action actions[],
1240                     bool external __rte_unused,
1241                     int hairpin __rte_unused,
1242                     struct rte_flow_error *error)
1243 {
1244         int ret;
1245         uint64_t action_flags = 0;
1246         uint64_t item_flags = 0;
1247         uint64_t last_item = 0;
1248         uint8_t next_protocol = 0xff;
1249         uint16_t ether_type = 0;
1250         char errstr[32];
1251
1252         if (items == NULL)
1253                 return -1;
1254         ret = mlx5_flow_validate_attributes(dev, attr, error);
1255         if (ret < 0)
1256                 return ret;
1257         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1258                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1259                 int ret = 0;
1260
1261                 switch (items->type) {
1262                 case RTE_FLOW_ITEM_TYPE_VOID:
1263                         break;
1264                 case RTE_FLOW_ITEM_TYPE_ETH:
1265                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1266                                                           false, error);
1267                         if (ret < 0)
1268                                 return ret;
1269                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1270                                              MLX5_FLOW_LAYER_OUTER_L2;
1271                         if (items->mask != NULL && items->spec != NULL) {
1272                                 ether_type =
1273                                         ((const struct rte_flow_item_eth *)
1274                                          items->spec)->type;
1275                                 ether_type &=
1276                                         ((const struct rte_flow_item_eth *)
1277                                          items->mask)->type;
1278                                 ether_type = rte_be_to_cpu_16(ether_type);
1279                         } else {
1280                                 ether_type = 0;
1281                         }
1282                         break;
1283                 case RTE_FLOW_ITEM_TYPE_VLAN:
1284                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1285                                                            dev, error);
1286                         if (ret < 0)
1287                                 return ret;
1288                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1289                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1290                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1291                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1292                         if (items->mask != NULL && items->spec != NULL) {
1293                                 ether_type =
1294                                         ((const struct rte_flow_item_vlan *)
1295                                          items->spec)->inner_type;
1296                                 ether_type &=
1297                                         ((const struct rte_flow_item_vlan *)
1298                                          items->mask)->inner_type;
1299                                 ether_type = rte_be_to_cpu_16(ether_type);
1300                         } else {
1301                                 ether_type = 0;
1302                         }
1303                         break;
1304                 case RTE_FLOW_ITEM_TYPE_IPV4:
1305                         ret = mlx5_flow_validate_item_ipv4
1306                                                 (items, item_flags,
1307                                                  last_item, ether_type, NULL,
1308                                                  MLX5_ITEM_RANGE_NOT_ACCEPTED,
1309                                                  error);
1310                         if (ret < 0)
1311                                 return ret;
1312                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1313                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1314                         if (items->mask != NULL &&
1315                             ((const struct rte_flow_item_ipv4 *)
1316                              items->mask)->hdr.next_proto_id) {
1317                                 next_protocol =
1318                                         ((const struct rte_flow_item_ipv4 *)
1319                                          (items->spec))->hdr.next_proto_id;
1320                                 next_protocol &=
1321                                         ((const struct rte_flow_item_ipv4 *)
1322                                          (items->mask))->hdr.next_proto_id;
1323                         } else {
1324                                 /* Reset for inner layer. */
1325                                 next_protocol = 0xff;
1326                         }
1327                         break;
1328                 case RTE_FLOW_ITEM_TYPE_IPV6:
1329                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1330                                                            last_item,
1331                                                            ether_type, NULL,
1332                                                            error);
1333                         if (ret < 0)
1334                                 return ret;
1335                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1336                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1337                         if (items->mask != NULL &&
1338                             ((const struct rte_flow_item_ipv6 *)
1339                              items->mask)->hdr.proto) {
1340                                 next_protocol =
1341                                         ((const struct rte_flow_item_ipv6 *)
1342                                          items->spec)->hdr.proto;
1343                                 next_protocol &=
1344                                         ((const struct rte_flow_item_ipv6 *)
1345                                          items->mask)->hdr.proto;
1346                         } else {
1347                                 /* Reset for inner layer. */
1348                                 next_protocol = 0xff;
1349                         }
1350                         break;
1351                 case RTE_FLOW_ITEM_TYPE_UDP:
1352                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1353                                                           next_protocol,
1354                                                           error);
1355                         if (ret < 0)
1356                                 return ret;
1357                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1358                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1359                         break;
1360                 case RTE_FLOW_ITEM_TYPE_TCP:
1361                         ret = mlx5_flow_validate_item_tcp
1362                                                 (items, item_flags,
1363                                                  next_protocol,
1364                                                  &rte_flow_item_tcp_mask,
1365                                                  error);
1366                         if (ret < 0)
1367                                 return ret;
1368                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1369                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1370                         break;
1371                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1372                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1373                                                             error);
1374                         if (ret < 0)
1375                                 return ret;
1376                         last_item = MLX5_FLOW_LAYER_VXLAN;
1377                         break;
1378                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1379                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1380                                                                 item_flags,
1381                                                                 dev, error);
1382                         if (ret < 0)
1383                                 return ret;
1384                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1385                         break;
1386                 case RTE_FLOW_ITEM_TYPE_GRE:
1387                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1388                                                           next_protocol, error);
1389                         if (ret < 0)
1390                                 return ret;
1391                         last_item = MLX5_FLOW_LAYER_GRE;
1392                         break;
1393                 case RTE_FLOW_ITEM_TYPE_MPLS:
1394                         ret = mlx5_flow_validate_item_mpls(dev, items,
1395                                                            item_flags,
1396                                                            last_item, error);
1397                         if (ret < 0)
1398                                 return ret;
1399                         last_item = MLX5_FLOW_LAYER_MPLS;
1400                         break;
1401                 default:
1402                         snprintf(errstr, sizeof(errstr), "item type %d not supported",
1403                                  items->type);
1404                         return rte_flow_error_set(error, ENOTSUP,
1405                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1406                                                   NULL, errstr);
1407                 }
1408                 item_flags |= last_item;
1409         }
1410         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1411                 switch (actions->type) {
1412                 case RTE_FLOW_ACTION_TYPE_VOID:
1413                         break;
1414                 case RTE_FLOW_ACTION_TYPE_FLAG:
1415                         ret = mlx5_flow_validate_action_flag(action_flags,
1416                                                              attr,
1417                                                              error);
1418                         if (ret < 0)
1419                                 return ret;
1420                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1421                         break;
1422                 case RTE_FLOW_ACTION_TYPE_MARK:
1423                         ret = mlx5_flow_validate_action_mark(actions,
1424                                                              action_flags,
1425                                                              attr,
1426                                                              error);
1427                         if (ret < 0)
1428                                 return ret;
1429                         action_flags |= MLX5_FLOW_ACTION_MARK;
1430                         break;
1431                 case RTE_FLOW_ACTION_TYPE_DROP:
1432                         ret = mlx5_flow_validate_action_drop(action_flags,
1433                                                              attr,
1434                                                              error);
1435                         if (ret < 0)
1436                                 return ret;
1437                         action_flags |= MLX5_FLOW_ACTION_DROP;
1438                         break;
1439                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1440                         ret = mlx5_flow_validate_action_queue(actions,
1441                                                               action_flags, dev,
1442                                                               attr,
1443                                                               error);
1444                         if (ret < 0)
1445                                 return ret;
1446                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1447                         break;
1448                 case RTE_FLOW_ACTION_TYPE_RSS:
1449                         ret = mlx5_flow_validate_action_rss(actions,
1450                                                             action_flags, dev,
1451                                                             attr, item_flags,
1452                                                             error);
1453                         if (ret < 0)
1454                                 return ret;
1455                         action_flags |= MLX5_FLOW_ACTION_RSS;
1456                         break;
1457                 case RTE_FLOW_ACTION_TYPE_COUNT:
1458                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1459                         if (ret < 0)
1460                                 return ret;
1461                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1462                         break;
1463                 default:
1464                         return rte_flow_error_set(error, ENOTSUP,
1465                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1466                                                   actions,
1467                                                   "action not supported");
1468                 }
1469         }
1470         /*
1471          * Validate the drop action mutual exclusion with other actions.
1472          * Drop action is mutually-exclusive with any other action, except for
1473          * Count action.
1474          */
1475         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1476             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1477                 return rte_flow_error_set(error, EINVAL,
1478                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1479                                           "Drop action is mutually-exclusive "
1480                                           "with any other action, except for "
1481                                           "Count action");
1482         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1483                 return rte_flow_error_set(error, EINVAL,
1484                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1485                                           "no fate action is found");
1486         return 0;
1487 }
1488
1489 /**
1490  * Calculate the required bytes that are needed for the action part of the verbs
1491  * flow.
1492  *
1493  * @param[in] actions
1494  *   Pointer to the list of actions.
1495  *
1496  * @return
1497  *   The size of the memory needed for all actions.
1498  */
1499 static int
1500 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1501 {
1502         int size = 0;
1503
1504         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1505                 switch (actions->type) {
1506                 case RTE_FLOW_ACTION_TYPE_VOID:
1507                         break;
1508                 case RTE_FLOW_ACTION_TYPE_FLAG:
1509                         size += sizeof(struct ibv_flow_spec_action_tag);
1510                         break;
1511                 case RTE_FLOW_ACTION_TYPE_MARK:
1512                         size += sizeof(struct ibv_flow_spec_action_tag);
1513                         break;
1514                 case RTE_FLOW_ACTION_TYPE_DROP:
1515                         size += sizeof(struct ibv_flow_spec_action_drop);
1516                         break;
1517                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1518                         break;
1519                 case RTE_FLOW_ACTION_TYPE_RSS:
1520                         break;
1521                 case RTE_FLOW_ACTION_TYPE_COUNT:
1522 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1523         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1524                         size += sizeof(struct ibv_flow_spec_counter_action);
1525 #endif
1526                         break;
1527                 default:
1528                         break;
1529                 }
1530         }
1531         return size;
1532 }
1533
1534 /**
1535  * Calculate the required bytes that are needed for the item part of the verbs
1536  * flow.
1537  *
1538  * @param[in] items
1539  *   Pointer to the list of items.
1540  *
1541  * @return
1542  *   The size of the memory needed for all items.
1543  */
1544 static int
1545 flow_verbs_get_items_size(const struct rte_flow_item items[])
1546 {
1547         int size = 0;
1548
1549         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1550                 switch (items->type) {
1551                 case RTE_FLOW_ITEM_TYPE_VOID:
1552                         break;
1553                 case RTE_FLOW_ITEM_TYPE_ETH:
1554                         size += sizeof(struct ibv_flow_spec_eth);
1555                         break;
1556                 case RTE_FLOW_ITEM_TYPE_VLAN:
1557                         size += sizeof(struct ibv_flow_spec_eth);
1558                         break;
1559                 case RTE_FLOW_ITEM_TYPE_IPV4:
1560                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1561                         break;
1562                 case RTE_FLOW_ITEM_TYPE_IPV6:
1563                         size += sizeof(struct ibv_flow_spec_ipv6);
1564                         break;
1565                 case RTE_FLOW_ITEM_TYPE_UDP:
1566                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1567                         break;
1568                 case RTE_FLOW_ITEM_TYPE_TCP:
1569                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1570                         break;
1571                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1572                         size += sizeof(struct ibv_flow_spec_tunnel);
1573                         break;
1574                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1575                         size += sizeof(struct ibv_flow_spec_tunnel);
1576                         break;
1577 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1578                 case RTE_FLOW_ITEM_TYPE_GRE:
1579                         size += sizeof(struct ibv_flow_spec_gre);
1580                         break;
1581                 case RTE_FLOW_ITEM_TYPE_MPLS:
1582                         size += sizeof(struct ibv_flow_spec_mpls);
1583                         break;
1584 #else
1585                 case RTE_FLOW_ITEM_TYPE_GRE:
1586                         size += sizeof(struct ibv_flow_spec_tunnel);
1587                         break;
1588 #endif
1589                 default:
1590                         break;
1591                 }
1592         }
1593         return size;
1594 }
1595
1596 /**
1597  * Internal preparation function. Allocate mlx5_flow with the required size.
1598  * The required size is calculate based on the actions and items. This function
1599  * also returns the detected actions and items for later use.
1600  *
1601  * @param[in] dev
1602  *   Pointer to Ethernet device.
1603  * @param[in] attr
1604  *   Pointer to the flow attributes.
1605  * @param[in] items
1606  *   Pointer to the list of items.
1607  * @param[in] actions
1608  *   Pointer to the list of actions.
1609  * @param[out] error
1610  *   Pointer to the error structure.
1611  *
1612  * @return
1613  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1614  *   is set.
1615  */
1616 static struct mlx5_flow *
1617 flow_verbs_prepare(struct rte_eth_dev *dev,
1618                    const struct rte_flow_attr *attr __rte_unused,
1619                    const struct rte_flow_item items[],
1620                    const struct rte_flow_action actions[],
1621                    struct rte_flow_error *error)
1622 {
1623         size_t size = 0;
1624         uint32_t handle_idx = 0;
1625         struct mlx5_flow *dev_flow;
1626         struct mlx5_flow_handle *dev_handle;
1627         struct mlx5_priv *priv = dev->data->dev_private;
1628         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1629
1630         MLX5_ASSERT(wks);
1631         size += flow_verbs_get_actions_size(actions);
1632         size += flow_verbs_get_items_size(items);
1633         if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1634                 rte_flow_error_set(error, E2BIG,
1635                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1636                                    "Verbs spec/action size too large");
1637                 return NULL;
1638         }
1639         /* In case of corrupting the memory. */
1640         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1641                 rte_flow_error_set(error, ENOSPC,
1642                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1643                                    "not free temporary device flow");
1644                 return NULL;
1645         }
1646         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1647                                    &handle_idx);
1648         if (!dev_handle) {
1649                 rte_flow_error_set(error, ENOMEM,
1650                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1651                                    "not enough memory to create flow handle");
1652                 return NULL;
1653         }
1654         MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows));
1655         dev_flow = &wks->flows[wks->flow_idx++];
1656         dev_flow->handle = dev_handle;
1657         dev_flow->handle_idx = handle_idx;
1658         /* Memcpy is used, only size needs to be cleared to 0. */
1659         dev_flow->verbs.size = 0;
1660         dev_flow->verbs.attr.num_of_specs = 0;
1661         dev_flow->ingress = attr->ingress;
1662         dev_flow->hash_fields = 0;
1663         /* Need to set transfer attribute: not supported in Verbs mode. */
1664         return dev_flow;
1665 }
1666
1667 /**
1668  * Fill the flow with verb spec.
1669  *
1670  * @param[in] dev
1671  *   Pointer to Ethernet device.
1672  * @param[in, out] dev_flow
1673  *   Pointer to the mlx5 flow.
1674  * @param[in] attr
1675  *   Pointer to the flow attributes.
1676  * @param[in] items
1677  *   Pointer to the list of items.
1678  * @param[in] actions
1679  *   Pointer to the list of actions.
1680  * @param[out] error
1681  *   Pointer to the error structure.
1682  *
1683  * @return
1684  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1685  */
1686 static int
1687 flow_verbs_translate(struct rte_eth_dev *dev,
1688                      struct mlx5_flow *dev_flow,
1689                      const struct rte_flow_attr *attr,
1690                      const struct rte_flow_item items[],
1691                      const struct rte_flow_action actions[],
1692                      struct rte_flow_error *error)
1693 {
1694         uint64_t item_flags = 0;
1695         uint64_t action_flags = 0;
1696         uint64_t priority = attr->priority;
1697         uint32_t subpriority = 0;
1698         struct mlx5_priv *priv = dev->data->dev_private;
1699         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1700         struct mlx5_flow_rss_desc *rss_desc;
1701         char errstr[32];
1702
1703         MLX5_ASSERT(wks);
1704         rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
1705         if (priority == MLX5_FLOW_PRIO_RSVD)
1706                 priority = priv->config.flow_prio - 1;
1707         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1708                 int ret;
1709
1710                 switch (actions->type) {
1711                 case RTE_FLOW_ACTION_TYPE_VOID:
1712                         break;
1713                 case RTE_FLOW_ACTION_TYPE_FLAG:
1714                         flow_verbs_translate_action_flag(dev_flow, actions);
1715                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1716                         dev_flow->handle->mark = 1;
1717                         break;
1718                 case RTE_FLOW_ACTION_TYPE_MARK:
1719                         flow_verbs_translate_action_mark(dev_flow, actions);
1720                         action_flags |= MLX5_FLOW_ACTION_MARK;
1721                         dev_flow->handle->mark = 1;
1722                         break;
1723                 case RTE_FLOW_ACTION_TYPE_DROP:
1724                         flow_verbs_translate_action_drop(dev_flow, actions);
1725                         action_flags |= MLX5_FLOW_ACTION_DROP;
1726                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
1727                         break;
1728                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1729                         flow_verbs_translate_action_queue(rss_desc, actions);
1730                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1731                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1732                         break;
1733                 case RTE_FLOW_ACTION_TYPE_RSS:
1734                         flow_verbs_translate_action_rss(rss_desc, actions);
1735                         action_flags |= MLX5_FLOW_ACTION_RSS;
1736                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1737                         break;
1738                 case RTE_FLOW_ACTION_TYPE_COUNT:
1739                         ret = flow_verbs_translate_action_count(dev_flow,
1740                                                                 actions,
1741                                                                 dev, error);
1742                         if (ret < 0)
1743                                 return ret;
1744                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1745                         break;
1746                 default:
1747                         return rte_flow_error_set(error, ENOTSUP,
1748                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1749                                                   actions,
1750                                                   "action not supported");
1751                 }
1752         }
1753         dev_flow->act_flags = action_flags;
1754         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1755                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1756
1757                 switch (items->type) {
1758                 case RTE_FLOW_ITEM_TYPE_VOID:
1759                         break;
1760                 case RTE_FLOW_ITEM_TYPE_ETH:
1761                         flow_verbs_translate_item_eth(dev_flow, items,
1762                                                       item_flags);
1763                         subpriority = MLX5_PRIORITY_MAP_L2;
1764                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1765                                                MLX5_FLOW_LAYER_OUTER_L2;
1766                         break;
1767                 case RTE_FLOW_ITEM_TYPE_VLAN:
1768                         flow_verbs_translate_item_vlan(dev_flow, items,
1769                                                        item_flags);
1770                         subpriority = MLX5_PRIORITY_MAP_L2;
1771                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1772                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1773                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1774                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1775                         break;
1776                 case RTE_FLOW_ITEM_TYPE_IPV4:
1777                         flow_verbs_translate_item_ipv4(dev_flow, items,
1778                                                        item_flags);
1779                         subpriority = MLX5_PRIORITY_MAP_L3;
1780                         dev_flow->hash_fields |=
1781                                 mlx5_flow_hashfields_adjust
1782                                         (rss_desc, tunnel,
1783                                          MLX5_IPV4_LAYER_TYPES,
1784                                          MLX5_IPV4_IBV_RX_HASH);
1785                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1786                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1787                         break;
1788                 case RTE_FLOW_ITEM_TYPE_IPV6:
1789                         flow_verbs_translate_item_ipv6(dev_flow, items,
1790                                                        item_flags);
1791                         subpriority = MLX5_PRIORITY_MAP_L3;
1792                         dev_flow->hash_fields |=
1793                                 mlx5_flow_hashfields_adjust
1794                                         (rss_desc, tunnel,
1795                                          MLX5_IPV6_LAYER_TYPES,
1796                                          MLX5_IPV6_IBV_RX_HASH);
1797                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1798                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1799                         break;
1800                 case RTE_FLOW_ITEM_TYPE_TCP:
1801                         flow_verbs_translate_item_tcp(dev_flow, items,
1802                                                       item_flags);
1803                         subpriority = MLX5_PRIORITY_MAP_L4;
1804                         dev_flow->hash_fields |=
1805                                 mlx5_flow_hashfields_adjust
1806                                         (rss_desc, tunnel, ETH_RSS_TCP,
1807                                          (IBV_RX_HASH_SRC_PORT_TCP |
1808                                           IBV_RX_HASH_DST_PORT_TCP));
1809                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1810                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1811                         break;
1812                 case RTE_FLOW_ITEM_TYPE_UDP:
1813                         flow_verbs_translate_item_udp(dev_flow, items,
1814                                                       item_flags);
1815                         subpriority = MLX5_PRIORITY_MAP_L4;
1816                         dev_flow->hash_fields |=
1817                                 mlx5_flow_hashfields_adjust
1818                                         (rss_desc, tunnel, ETH_RSS_UDP,
1819                                          (IBV_RX_HASH_SRC_PORT_UDP |
1820                                           IBV_RX_HASH_DST_PORT_UDP));
1821                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1822                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1823                         break;
1824                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1825                         flow_verbs_translate_item_vxlan(dev_flow, items,
1826                                                         item_flags);
1827                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1828                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1829                         break;
1830                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1831                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1832                                                             item_flags);
1833                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1834                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1835                         break;
1836                 case RTE_FLOW_ITEM_TYPE_GRE:
1837                         flow_verbs_translate_item_gre(dev_flow, items,
1838                                                       item_flags);
1839                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1840                         item_flags |= MLX5_FLOW_LAYER_GRE;
1841                         break;
1842                 case RTE_FLOW_ITEM_TYPE_MPLS:
1843                         flow_verbs_translate_item_mpls(dev_flow, items,
1844                                                        item_flags);
1845                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1846                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1847                         break;
1848                 default:
1849                         snprintf(errstr, sizeof(errstr), "item type %d not supported",
1850                                  items->type);
1851                         return rte_flow_error_set(error, ENOTSUP,
1852                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1853                                                   NULL, errstr);
1854                 }
1855         }
1856         dev_flow->handle->layers = item_flags;
1857         /* Other members of attr will be ignored. */
1858         dev_flow->verbs.attr.priority =
1859                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1860         dev_flow->verbs.attr.port = (uint8_t)priv->dev_port;
1861         return 0;
1862 }
1863
1864 /**
1865  * Remove the flow from the NIC but keeps it in memory.
1866  *
1867  * @param[in] dev
1868  *   Pointer to the Ethernet device structure.
1869  * @param[in, out] flow
1870  *   Pointer to flow structure.
1871  */
1872 static void
1873 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1874 {
1875         struct mlx5_priv *priv = dev->data->dev_private;
1876         struct mlx5_flow_handle *handle;
1877         uint32_t handle_idx;
1878
1879         if (!flow)
1880                 return;
1881         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1882                        handle_idx, handle, next) {
1883                 if (handle->drv_flow) {
1884                         claim_zero(mlx5_glue->destroy_flow(handle->drv_flow));
1885                         handle->drv_flow = NULL;
1886                 }
1887                 /* hrxq is union, don't touch it only the flag is set. */
1888                 if (handle->rix_hrxq &&
1889                     handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
1890                         mlx5_hrxq_release(dev, handle->rix_hrxq);
1891                         handle->rix_hrxq = 0;
1892                 }
1893                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1894                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1895         }
1896 }
1897
1898 /**
1899  * Remove the flow from the NIC and the memory.
1900  *
1901  * @param[in] dev
1902  *   Pointer to the Ethernet device structure.
1903  * @param[in, out] flow
1904  *   Pointer to flow structure.
1905  */
1906 static void
1907 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1908 {
1909         struct mlx5_priv *priv = dev->data->dev_private;
1910         struct mlx5_flow_handle *handle;
1911
1912         if (!flow)
1913                 return;
1914         flow_verbs_remove(dev, flow);
1915         while (flow->dev_handles) {
1916                 uint32_t tmp_idx = flow->dev_handles;
1917
1918                 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1919                                    tmp_idx);
1920                 if (!handle)
1921                         return;
1922                 flow->dev_handles = handle->next.next;
1923                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1924                            tmp_idx);
1925         }
1926         if (flow->counter) {
1927                 flow_verbs_counter_release(dev, flow->counter);
1928                 flow->counter = 0;
1929         }
1930 }
1931
1932 /**
1933  * Apply the flow to the NIC.
1934  *
1935  * @param[in] dev
1936  *   Pointer to the Ethernet device structure.
1937  * @param[in, out] flow
1938  *   Pointer to flow structure.
1939  * @param[out] error
1940  *   Pointer to error structure.
1941  *
1942  * @return
1943  *   0 on success, a negative errno value otherwise and rte_errno is set.
1944  */
1945 static int
1946 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1947                  struct rte_flow_error *error)
1948 {
1949         struct mlx5_priv *priv = dev->data->dev_private;
1950         struct mlx5_flow_handle *handle;
1951         struct mlx5_flow *dev_flow;
1952         struct mlx5_hrxq *hrxq;
1953         uint32_t dev_handles;
1954         int err;
1955         int idx;
1956         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
1957
1958         MLX5_ASSERT(wks);
1959         for (idx = wks->flow_idx - 1; idx >= wks->flow_nested_idx; idx--) {
1960                 dev_flow = &wks->flows[idx];
1961                 handle = dev_flow->handle;
1962                 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1963                         MLX5_ASSERT(priv->drop_queue.hrxq);
1964                         hrxq = priv->drop_queue.hrxq;
1965                 } else {
1966                         uint32_t hrxq_idx;
1967                         struct mlx5_flow_rss_desc *rss_desc =
1968                                 &wks->rss_desc[!!wks->flow_nested_idx];
1969
1970                         MLX5_ASSERT(rss_desc->queue_num);
1971                         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
1972                         rss_desc->hash_fields = dev_flow->hash_fields;
1973                         rss_desc->tunnel = !!(handle->layers &
1974                                               MLX5_FLOW_LAYER_TUNNEL);
1975                         rss_desc->standalone = false;
1976                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
1977                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1978                                               hrxq_idx);
1979                         if (!hrxq) {
1980                                 rte_flow_error_set
1981                                         (error, rte_errno,
1982                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1983                                          "cannot get hash queue");
1984                                 goto error;
1985                         }
1986                         handle->rix_hrxq = hrxq_idx;
1987                 }
1988                 MLX5_ASSERT(hrxq);
1989                 handle->drv_flow = mlx5_glue->create_flow
1990                                         (hrxq->qp, &dev_flow->verbs.attr);
1991                 if (!handle->drv_flow) {
1992                         rte_flow_error_set(error, errno,
1993                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1994                                            NULL,
1995                                            "hardware refuses to create flow");
1996                         goto error;
1997                 }
1998                 if (priv->vmwa_context &&
1999                     handle->vf_vlan.tag && !handle->vf_vlan.created) {
2000                         /*
2001                          * The rule contains the VLAN pattern.
2002                          * For VF we are going to create VLAN
2003                          * interface to make hypervisor set correct
2004                          * e-Switch vport context.
2005                          */
2006                         mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
2007                 }
2008         }
2009         return 0;
2010 error:
2011         err = rte_errno; /* Save rte_errno before cleanup. */
2012         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2013                        dev_handles, handle, next) {
2014                 /* hrxq is union, don't touch it only the flag is set. */
2015                 if (handle->rix_hrxq &&
2016                     handle->fate_action == MLX5_FLOW_FATE_QUEUE) {
2017                         mlx5_hrxq_release(dev, handle->rix_hrxq);
2018                         handle->rix_hrxq = 0;
2019                 }
2020                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
2021                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
2022         }
2023         rte_errno = err; /* Restore rte_errno. */
2024         return -rte_errno;
2025 }
2026
2027 /**
2028  * Query a flow.
2029  *
2030  * @see rte_flow_query()
2031  * @see rte_flow_ops
2032  */
2033 static int
2034 flow_verbs_query(struct rte_eth_dev *dev,
2035                  struct rte_flow *flow,
2036                  const struct rte_flow_action *actions,
2037                  void *data,
2038                  struct rte_flow_error *error)
2039 {
2040         int ret = -EINVAL;
2041
2042         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2043                 switch (actions->type) {
2044                 case RTE_FLOW_ACTION_TYPE_VOID:
2045                         break;
2046                 case RTE_FLOW_ACTION_TYPE_COUNT:
2047                         ret = flow_verbs_counter_query(dev, flow, data, error);
2048                         break;
2049                 default:
2050                         return rte_flow_error_set(error, ENOTSUP,
2051                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2052                                                   actions,
2053                                                   "action not supported");
2054                 }
2055         }
2056         return ret;
2057 }
2058
2059 static int
2060 flow_verbs_sync_domain(struct rte_eth_dev *dev, uint32_t domains,
2061                        uint32_t flags)
2062 {
2063         RTE_SET_USED(dev);
2064         RTE_SET_USED(domains);
2065         RTE_SET_USED(flags);
2066
2067         return 0;
2068 }
2069
2070 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
2071         .validate = flow_verbs_validate,
2072         .prepare = flow_verbs_prepare,
2073         .translate = flow_verbs_translate,
2074         .apply = flow_verbs_apply,
2075         .remove = flow_verbs_remove,
2076         .destroy = flow_verbs_destroy,
2077         .query = flow_verbs_query,
2078         .sync_domain = flow_verbs_sync_domain,
2079 };