drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_ip.h>
  18
  19 #include <mlx5_glue.h>
  20 #include <mlx5_prm.h>
  21 #include <mlx5_malloc.h>
  22
  23 #include "mlx5_defs.h"
  24 #include "mlx5.h"
  25 #include "mlx5_flow.h"
  26 #include "mlx5_rxtx.h"
  27
  28 #define VERBS_SPEC_INNER(item_flags) \
  29         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  30
  31 /* Map of Verbs to Flow priority with 8 Verbs priorities. */
  32 static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = {
  33         { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 },
  34 };
  35
  36 /* Map of Verbs to Flow priority with 16 Verbs priorities. */
  37 static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = {
  38         { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 },
  39         { 9, 10, 11 }, { 12, 13, 14 },
  40 };
  41
  42 /**
  43  * Discover the maximum number of priority available.
  44  *
  45  * @param[in] dev
  46  *   Pointer to the Ethernet device structure.
  47  *
  48  * @return
  49  *   number of supported flow priority on success, a negative errno
  50  *   value otherwise and rte_errno is set.
  51  */
  52 int
  53 mlx5_flow_discover_priorities(struct rte_eth_dev *dev)
  54 {
  55         struct mlx5_priv *priv = dev->data->dev_private;
  56         struct {
  57                 struct ibv_flow_attr attr;
  58                 struct ibv_flow_spec_eth eth;
  59                 struct ibv_flow_spec_action_drop drop;
  60         } flow_attr = {
  61                 .attr = {
  62                         .num_of_specs = 2,
  63                         .port = (uint8_t)priv->dev_port,
  64                 },
  65                 .eth = {
  66                         .type = IBV_FLOW_SPEC_ETH,
  67                         .size = sizeof(struct ibv_flow_spec_eth),
  68                 },
  69                 .drop = {
  70                         .size = sizeof(struct ibv_flow_spec_action_drop),
  71                         .type = IBV_FLOW_SPEC_ACTION_DROP,
  72                 },
  73         };
  74         struct ibv_flow *flow;
  75         struct mlx5_hrxq *drop = mlx5_drop_action_create(dev);
  76         uint16_t vprio[] = { 8, 16 };
  77         int i;
  78         int priority = 0;
  79
  80         if (!drop) {
  81                 rte_errno = ENOTSUP;
  82                 return -rte_errno;
  83         }
  84         for (i = 0; i != RTE_DIM(vprio); i++) {
  85                 flow_attr.attr.priority = vprio[i] - 1;
  86                 flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr);
  87                 if (!flow)
  88                         break;
  89                 claim_zero(mlx5_glue->destroy_flow(flow));
  90                 priority = vprio[i];
  91         }
  92         mlx5_drop_action_destroy(dev);
  93         switch (priority) {
  94         case 8:
  95                 priority = RTE_DIM(priority_map_3);
  96                 break;
  97         case 16:
  98                 priority = RTE_DIM(priority_map_5);
  99                 break;
 100         default:
 101                 rte_errno = ENOTSUP;
 102                 DRV_LOG(ERR,
 103                         "port %u verbs maximum priority: %d expected 8/16",
 104                         dev->data->port_id, priority);
 105                 return -rte_errno;
 106         }
 107         DRV_LOG(INFO, "port %u flow maximum priority: %d",
 108                 dev->data->port_id, priority);
 109         return priority;
 110 }
 111
 112 /**
 113  * Adjust flow priority based on the highest layer and the request priority.
 114  *
 115  * @param[in] dev
 116  *   Pointer to the Ethernet device structure.
 117  * @param[in] priority
 118  *   The rule base priority.
 119  * @param[in] subpriority
 120  *   The priority based on the items.
 121  *
 122  * @return
 123  *   The new priority.
 124  */
 125 uint32_t
 126 mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 127                                    uint32_t subpriority)
 128 {
 129         uint32_t res = 0;
 130         struct mlx5_priv *priv = dev->data->dev_private;
 131
 132         switch (priv->config.flow_prio) {
 133         case RTE_DIM(priority_map_3):
 134                 res = priority_map_3[priority][subpriority];
 135                 break;
 136         case RTE_DIM(priority_map_5):
 137                 res = priority_map_5[priority][subpriority];
 138                 break;
 139         }
 140         return  res;
 141 }
 142
 143 /**
 144  * Get Verbs flow counter by index.
 145  *
 146  * @param[in] dev
 147  *   Pointer to the Ethernet device structure.
 148  * @param[in] idx
 149  *   mlx5 flow counter index in the container.
 150  * @param[out] ppool
 151  *   mlx5 flow counter pool in the container,
 152  *
 153  * @return
 154  *   A pointer to the counter, NULL otherwise.
 155  */
 156 static struct mlx5_flow_counter *
 157 flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev,
 158                               uint32_t idx,
 159                               struct mlx5_flow_counter_pool **ppool)
 160 {
 161         struct mlx5_priv *priv = dev->data->dev_private;
 162         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 163         struct mlx5_flow_counter_pool *pool;
 164
 165         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
 166         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
 167         MLX5_ASSERT(pool);
 168         if (ppool)
 169                 *ppool = pool;
 170         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
 171 }
 172
 173 /**
 174  * Create Verbs flow counter with Verbs library.
 175  *
 176  * @param[in] dev
 177  *   Pointer to the Ethernet device structure.
 178  * @param[in, out] counter
 179  *   mlx5 flow counter object, contains the counter id,
 180  *   handle of created Verbs flow counter is returned
 181  *   in cs field (if counters are supported).
 182  *
 183  * @return
 184  *   0 On success else a negative errno value is returned
 185  *   and rte_errno is set.
 186  */
 187 static int
 188 flow_verbs_counter_create(struct rte_eth_dev *dev,
 189                           struct mlx5_flow_counter *counter)
 190 {
 191 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 192         struct mlx5_priv *priv = dev->data->dev_private;
 193         struct ibv_context *ctx = priv->sh->ctx;
 194         struct ibv_counter_set_init_attr init = {
 195                          .counter_set_id = counter->shared_info.id};
 196
 197         counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init);
 198         if (!counter->dcs_when_free) {
 199                 rte_errno = ENOTSUP;
 200                 return -ENOTSUP;
 201         }
 202         return 0;
 203 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 204         struct mlx5_priv *priv = dev->data->dev_private;
 205         struct ibv_context *ctx = priv->sh->ctx;
 206         struct ibv_counters_init_attr init = {0};
 207         struct ibv_counter_attach_attr attach;
 208         int ret;
 209
 210         memset(&attach, 0, sizeof(attach));
 211         counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init);
 212         if (!counter->dcs_when_free) {
 213                 rte_errno = ENOTSUP;
 214                 return -ENOTSUP;
 215         }
 216         attach.counter_desc = IBV_COUNTER_PACKETS;
 217         attach.index = 0;
 218         ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL);
 219         if (!ret) {
 220                 attach.counter_desc = IBV_COUNTER_BYTES;
 221                 attach.index = 1;
 222                 ret = mlx5_glue->attach_counters
 223                                         (counter->dcs_when_free, &attach, NULL);
 224         }
 225         if (ret) {
 226                 claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free));
 227                 counter->dcs_when_free = NULL;
 228                 rte_errno = ret;
 229                 return -ret;
 230         }
 231         return 0;
 232 #else
 233         (void)dev;
 234         (void)counter;
 235         rte_errno = ENOTSUP;
 236         return -ENOTSUP;
 237 #endif
 238 }
 239
 240 /**
 241  * Get a flow counter.
 242  *
 243  * @param[in] dev
 244  *   Pointer to the Ethernet device structure.
 245  * @param[in] shared
 246  *   Indicate if this counter is shared with other flows.
 247  * @param[in] id
 248  *   Counter identifier.
 249  *
 250  * @return
 251  *   Index to the counter, 0 otherwise and rte_errno is set.
 252  */
 253 static uint32_t
 254 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 255 {
 256         struct mlx5_priv *priv = dev->data->dev_private;
 257         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
 258         struct mlx5_flow_counter_pool *pool = NULL;
 259         struct mlx5_flow_counter *cnt = NULL;
 260         union mlx5_l3t_data data;
 261         uint32_t n_valid = cmng->n_valid;
 262         uint32_t pool_idx, cnt_idx;
 263         uint32_t i;
 264         int ret;
 265
 266         if (shared && !mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) &&
 267             data.dword)
 268                 return data.dword;
 269         for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) {
 270                 pool = cmng->pools[pool_idx];
 271                 if (!pool)
 272                         continue;
 273                 cnt = TAILQ_FIRST(&pool->counters[0]);
 274                 if (cnt)
 275                         break;
 276         }
 277         if (!cnt) {
 278                 struct mlx5_flow_counter_pool **pools;
 279                 uint32_t size;
 280
 281                 if (n_valid == cmng->n) {
 282                         /* Resize the container pool array. */
 283                         size = sizeof(struct mlx5_flow_counter_pool *) *
 284                                      (n_valid + MLX5_CNT_CONTAINER_RESIZE);
 285                         pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0,
 286                                             SOCKET_ID_ANY);
 287                         if (!pools)
 288                                 return 0;
 289                         if (n_valid) {
 290                                 memcpy(pools, cmng->pools,
 291                                        sizeof(struct mlx5_flow_counter_pool *) *
 292                                        n_valid);
 293                                 mlx5_free(cmng->pools);
 294                         }
 295                         cmng->pools = pools;
 296                         cmng->n += MLX5_CNT_CONTAINER_RESIZE;
 297                 }
 298                 /* Allocate memory for new pool*/
 299                 size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL;
 300                 pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
 301                 if (!pool)
 302                         return 0;
 303                 for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
 304                         cnt = MLX5_POOL_GET_CNT(pool, i);
 305                         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 306                 }
 307                 cnt = MLX5_POOL_GET_CNT(pool, 0);
 308                 cmng->pools[n_valid] = pool;
 309                 pool_idx = n_valid;
 310                 cmng->n_valid++;
 311         }
 312         TAILQ_REMOVE(&pool->counters[0], cnt, next);
 313         i = MLX5_CNT_ARRAY_IDX(pool, cnt);
 314         cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i);
 315         if (shared) {
 316                 data.dword = cnt_idx;
 317                 if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
 318                         return 0;
 319                 cnt->shared_info.id = id;
 320                 cnt_idx |= MLX5_CNT_SHARED_OFFSET;
 321         }
 322         /* Create counter with Verbs. */
 323         ret = flow_verbs_counter_create(dev, cnt);
 324         if (!ret) {
 325                 cnt->dcs_when_active = cnt->dcs_when_free;
 326                 cnt->hits = 0;
 327                 cnt->bytes = 0;
 328                 return cnt_idx;
 329         }
 330         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 331         /* Some error occurred in Verbs library. */
 332         rte_errno = -ret;
 333         return 0;
 334 }
 335
 336 /**
 337  * Release a flow counter.
 338  *
 339  * @param[in] dev
 340  *   Pointer to the Ethernet device structure.
 341  * @param[in] counter
 342  *   Index to the counter handler.
 343  */
 344 static void
 345 flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter)
 346 {
 347         struct mlx5_priv *priv = dev->data->dev_private;
 348         struct mlx5_flow_counter_pool *pool;
 349         struct mlx5_flow_counter *cnt;
 350
 351         cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool);
 352         if (IS_SHARED_CNT(counter) &&
 353             mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
 354                 return;
 355 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 356         claim_zero(mlx5_glue->destroy_counter_set
 357                         ((struct ibv_counter_set *)cnt->dcs_when_active));
 358 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 359         claim_zero(mlx5_glue->destroy_counters
 360                                 ((struct ibv_counters *)cnt->dcs_when_active));
 361 #endif
 362         TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next);
 363 }
 364
 365 /**
 366  * Query a flow counter via Verbs library call.
 367  *
 368  * @see rte_flow_query()
 369  * @see rte_flow_ops
 370  */
 371 static int
 372 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 373                          struct rte_flow *flow, void *data,
 374                          struct rte_flow_error *error)
 375 {
 376 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 377         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 378         if (flow->counter) {
 379                 struct mlx5_flow_counter_pool *pool;
 380                 struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx
 381                                                 (dev, flow->counter, &pool);
 382                 struct rte_flow_query_count *qc = data;
 383                 uint64_t counters[2] = {0, 0};
 384 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 385                 struct ibv_query_counter_set_attr query_cs_attr = {
 386                         .dcs_when_free = (struct ibv_counter_set *)
 387                                                 cnt->dcs_when_active,
 388                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 389                 };
 390                 struct ibv_counter_set_data query_out = {
 391                         .out = counters,
 392                         .outlen = 2 * sizeof(uint64_t),
 393                 };
 394                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 395                                                        &query_out);
 396 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 397                 int err = mlx5_glue->query_counters
 398                         ((struct ibv_counters *)cnt->dcs_when_active, counters,
 399                                 RTE_DIM(counters),
 400                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 401 #endif
 402                 if (err)
 403                         return rte_flow_error_set
 404                                 (error, err,
 405                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 406                                  NULL,
 407                                  "cannot read counter");
 408                 qc->hits_set = 1;
 409                 qc->bytes_set = 1;
 410                 qc->hits = counters[0] - cnt->hits;
 411                 qc->bytes = counters[1] - cnt->bytes;
 412                 if (qc->reset) {
 413                         cnt->hits = counters[0];
 414                         cnt->bytes = counters[1];
 415                 }
 416                 return 0;
 417         }
 418         return rte_flow_error_set(error, EINVAL,
 419                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 420                                   NULL,
 421                                   "flow does not have counter");
 422 #else
 423         (void)flow;
 424         (void)data;
 425         return rte_flow_error_set(error, ENOTSUP,
 426                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 427                                   NULL,
 428                                   "counters are not available");
 429 #endif
 430 }
 431
 432 /**
 433  * Add a verbs item specification into @p verbs.
 434  *
 435  * @param[out] verbs
 436  *   Pointer to verbs structure.
 437  * @param[in] src
 438  *   Create specification.
 439  * @param[in] size
 440  *   Size in bytes of the specification to copy.
 441  */
 442 static void
 443 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
 444                     void *src, unsigned int size)
 445 {
 446         void *dst;
 447
 448         if (!verbs)
 449                 return;
 450         MLX5_ASSERT(verbs->specs);
 451         dst = (void *)(verbs->specs + verbs->size);
 452         memcpy(dst, src, size);
 453         ++verbs->attr.num_of_specs;
 454         verbs->size += size;
 455 }
 456
 457 /**
 458  * Convert the @p item into a Verbs specification. This function assumes that
 459  * the input is valid and that there is space to insert the requested item
 460  * into the flow.
 461  *
 462  * @param[in, out] dev_flow
 463  *   Pointer to dev_flow structure.
 464  * @param[in] item
 465  *   Item specification.
 466  * @param[in] item_flags
 467  *   Parsed item flags.
 468  */
 469 static void
 470 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 471                               const struct rte_flow_item *item,
 472                               uint64_t item_flags)
 473 {
 474         const struct rte_flow_item_eth *spec = item->spec;
 475         const struct rte_flow_item_eth *mask = item->mask;
 476         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 477         struct ibv_flow_spec_eth eth = {
 478                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 479                 .size = size,
 480         };
 481
 482         if (!mask)
 483                 mask = &rte_flow_item_eth_mask;
 484         if (spec) {
 485                 unsigned int i;
 486
 487                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 488                         RTE_ETHER_ADDR_LEN);
 489                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 490                         RTE_ETHER_ADDR_LEN);
 491                 eth.val.ether_type = spec->type;
 492                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 493                         RTE_ETHER_ADDR_LEN);
 494                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 495                         RTE_ETHER_ADDR_LEN);
 496                 eth.mask.ether_type = mask->type;
 497                 /* Remove unwanted bits from values. */
 498                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 499                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 500                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 501                 }
 502                 eth.val.ether_type &= eth.mask.ether_type;
 503         }
 504         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 505 }
 506
 507 /**
 508  * Update the VLAN tag in the Verbs Ethernet specification.
 509  * This function assumes that the input is valid and there is space to add
 510  * the requested item.
 511  *
 512  * @param[in, out] attr
 513  *   Pointer to Verbs attributes structure.
 514  * @param[in] eth
 515  *   Verbs structure containing the VLAN information to copy.
 516  */
 517 static void
 518 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 519                             struct ibv_flow_spec_eth *eth)
 520 {
 521         unsigned int i;
 522         const enum ibv_flow_spec_type search = eth->type;
 523         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 524                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 525
 526         for (i = 0; i != attr->num_of_specs; ++i) {
 527                 if (hdr->type == search) {
 528                         struct ibv_flow_spec_eth *e =
 529                                 (struct ibv_flow_spec_eth *)hdr;
 530
 531                         e->val.vlan_tag = eth->val.vlan_tag;
 532                         e->mask.vlan_tag = eth->mask.vlan_tag;
 533                         e->val.ether_type = eth->val.ether_type;
 534                         e->mask.ether_type = eth->mask.ether_type;
 535                         break;
 536                 }
 537                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 538         }
 539 }
 540
 541 /**
 542  * Convert the @p item into a Verbs specification. This function assumes that
 543  * the input is valid and that there is space to insert the requested item
 544  * into the flow.
 545  *
 546  * @param[in, out] dev_flow
 547  *   Pointer to dev_flow structure.
 548  * @param[in] item
 549  *   Item specification.
 550  * @param[in] item_flags
 551  *   Parsed item flags.
 552  */
 553 static void
 554 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 555                                const struct rte_flow_item *item,
 556                                uint64_t item_flags)
 557 {
 558         const struct rte_flow_item_vlan *spec = item->spec;
 559         const struct rte_flow_item_vlan *mask = item->mask;
 560         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 561         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 562         struct ibv_flow_spec_eth eth = {
 563                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 564                 .size = size,
 565         };
 566         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 567                                       MLX5_FLOW_LAYER_OUTER_L2;
 568
 569         if (!mask)
 570                 mask = &rte_flow_item_vlan_mask;
 571         if (spec) {
 572                 eth.val.vlan_tag = spec->tci;
 573                 eth.mask.vlan_tag = mask->tci;
 574                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 575                 eth.val.ether_type = spec->inner_type;
 576                 eth.mask.ether_type = mask->inner_type;
 577                 eth.val.ether_type &= eth.mask.ether_type;
 578         }
 579         if (!(item_flags & l2m))
 580                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 581         else
 582                 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, &eth);
 583         if (!tunnel)
 584                 dev_flow->handle->vf_vlan.tag =
 585                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 586 }
 587
 588 /**
 589  * Convert the @p item into a Verbs specification. This function assumes that
 590  * the input is valid and that there is space to insert the requested item
 591  * into the flow.
 592  *
 593  * @param[in, out] dev_flow
 594  *   Pointer to dev_flow structure.
 595  * @param[in] item
 596  *   Item specification.
 597  * @param[in] item_flags
 598  *   Parsed item flags.
 599  */
 600 static void
 601 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 602                                const struct rte_flow_item *item,
 603                                uint64_t item_flags)
 604 {
 605         const struct rte_flow_item_ipv4 *spec = item->spec;
 606         const struct rte_flow_item_ipv4 *mask = item->mask;
 607         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 608         struct ibv_flow_spec_ipv4_ext ipv4 = {
 609                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 610                 .size = size,
 611         };
 612
 613         if (!mask)
 614                 mask = &rte_flow_item_ipv4_mask;
 615         if (spec) {
 616                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 617                         .src_ip = spec->hdr.src_addr,
 618                         .dst_ip = spec->hdr.dst_addr,
 619                         .proto = spec->hdr.next_proto_id,
 620                         .tos = spec->hdr.type_of_service,
 621                 };
 622                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 623                         .src_ip = mask->hdr.src_addr,
 624                         .dst_ip = mask->hdr.dst_addr,
 625                         .proto = mask->hdr.next_proto_id,
 626                         .tos = mask->hdr.type_of_service,
 627                 };
 628                 /* Remove unwanted bits from values. */
 629                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 630                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 631                 ipv4.val.proto &= ipv4.mask.proto;
 632                 ipv4.val.tos &= ipv4.mask.tos;
 633         }
 634         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 635 }
 636
 637 /**
 638  * Convert the @p item into a Verbs specification. This function assumes that
 639  * the input is valid and that there is space to insert the requested item
 640  * into the flow.
 641  *
 642  * @param[in, out] dev_flow
 643  *   Pointer to dev_flow structure.
 644  * @param[in] item
 645  *   Item specification.
 646  * @param[in] item_flags
 647  *   Parsed item flags.
 648  */
 649 static void
 650 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 651                                const struct rte_flow_item *item,
 652                                uint64_t item_flags)
 653 {
 654         const struct rte_flow_item_ipv6 *spec = item->spec;
 655         const struct rte_flow_item_ipv6 *mask = item->mask;
 656         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 657         struct ibv_flow_spec_ipv6 ipv6 = {
 658                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 659                 .size = size,
 660         };
 661
 662         if (!mask)
 663                 mask = &rte_flow_item_ipv6_mask;
 664         if (spec) {
 665                 unsigned int i;
 666                 uint32_t vtc_flow_val;
 667                 uint32_t vtc_flow_mask;
 668
 669                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 670                        RTE_DIM(ipv6.val.src_ip));
 671                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 672                        RTE_DIM(ipv6.val.dst_ip));
 673                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 674                        RTE_DIM(ipv6.mask.src_ip));
 675                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 676                        RTE_DIM(ipv6.mask.dst_ip));
 677                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 678                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 679                 ipv6.val.flow_label =
 680                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 681                                          RTE_IPV6_HDR_FL_SHIFT);
 682                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 683                                          RTE_IPV6_HDR_TC_SHIFT;
 684                 ipv6.val.next_hdr = spec->hdr.proto;
 685                 ipv6.mask.flow_label =
 686                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 687                                          RTE_IPV6_HDR_FL_SHIFT);
 688                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 689                                           RTE_IPV6_HDR_TC_SHIFT;
 690                 ipv6.mask.next_hdr = mask->hdr.proto;
 691                 /* Remove unwanted bits from values. */
 692                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 693                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 694                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 695                 }
 696                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 697                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 698                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 699         }
 700         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 701 }
 702
 703 /**
 704  * Convert the @p item into a Verbs specification. This function assumes that
 705  * the input is valid and that there is space to insert the requested item
 706  * into the flow.
 707  *
 708  * @param[in, out] dev_flow
 709  *   Pointer to dev_flow structure.
 710  * @param[in] item
 711  *   Item specification.
 712  * @param[in] item_flags
 713  *   Parsed item flags.
 714  */
 715 static void
 716 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 717                               const struct rte_flow_item *item,
 718                               uint64_t item_flags __rte_unused)
 719 {
 720         const struct rte_flow_item_tcp *spec = item->spec;
 721         const struct rte_flow_item_tcp *mask = item->mask;
 722         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 723         struct ibv_flow_spec_tcp_udp tcp = {
 724                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 725                 .size = size,
 726         };
 727
 728         if (!mask)
 729                 mask = &rte_flow_item_tcp_mask;
 730         if (spec) {
 731                 tcp.val.dst_port = spec->hdr.dst_port;
 732                 tcp.val.src_port = spec->hdr.src_port;
 733                 tcp.mask.dst_port = mask->hdr.dst_port;
 734                 tcp.mask.src_port = mask->hdr.src_port;
 735                 /* Remove unwanted bits from values. */
 736                 tcp.val.src_port &= tcp.mask.src_port;
 737                 tcp.val.dst_port &= tcp.mask.dst_port;
 738         }
 739         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 740 }
 741
 742 /**
 743  * Convert the @p item into a Verbs specification. This function assumes that
 744  * the input is valid and that there is space to insert the requested item
 745  * into the flow.
 746  *
 747  * @param[in, out] dev_flow
 748  *   Pointer to dev_flow structure.
 749  * @param[in] item
 750  *   Item specification.
 751  * @param[in] item_flags
 752  *   Parsed item flags.
 753  */
 754 static void
 755 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 756                               const struct rte_flow_item *item,
 757                               uint64_t item_flags __rte_unused)
 758 {
 759         const struct rte_flow_item_udp *spec = item->spec;
 760         const struct rte_flow_item_udp *mask = item->mask;
 761         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 762         struct ibv_flow_spec_tcp_udp udp = {
 763                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 764                 .size = size,
 765         };
 766
 767         if (!mask)
 768                 mask = &rte_flow_item_udp_mask;
 769         if (spec) {
 770                 udp.val.dst_port = spec->hdr.dst_port;
 771                 udp.val.src_port = spec->hdr.src_port;
 772                 udp.mask.dst_port = mask->hdr.dst_port;
 773                 udp.mask.src_port = mask->hdr.src_port;
 774                 /* Remove unwanted bits from values. */
 775                 udp.val.src_port &= udp.mask.src_port;
 776                 udp.val.dst_port &= udp.mask.dst_port;
 777         }
 778         item++;
 779         while (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 780                 item++;
 781         if (!(udp.val.dst_port & udp.mask.dst_port)) {
 782                 switch ((item)->type) {
 783                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 784                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN);
 785                         udp.mask.dst_port = 0xffff;
 786                         break;
 787                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 788                         udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE);
 789                         udp.mask.dst_port = 0xffff;
 790                         break;
 791                 case RTE_FLOW_ITEM_TYPE_MPLS:
 792                         udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS);
 793                         udp.mask.dst_port = 0xffff;
 794                         break;
 795                 default:
 796                         break;
 797                 }
 798         }
 799
 800         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 801 }
 802
 803 /**
 804  * Convert the @p item into a Verbs specification. This function assumes that
 805  * the input is valid and that there is space to insert the requested item
 806  * into the flow.
 807  *
 808  * @param[in, out] dev_flow
 809  *   Pointer to dev_flow structure.
 810  * @param[in] item
 811  *   Item specification.
 812  * @param[in] item_flags
 813  *   Parsed item flags.
 814  */
 815 static void
 816 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 817                                 const struct rte_flow_item *item,
 818                                 uint64_t item_flags __rte_unused)
 819 {
 820         const struct rte_flow_item_vxlan *spec = item->spec;
 821         const struct rte_flow_item_vxlan *mask = item->mask;
 822         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 823         struct ibv_flow_spec_tunnel vxlan = {
 824                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 825                 .size = size,
 826         };
 827         union vni {
 828                 uint32_t vlan_id;
 829                 uint8_t vni[4];
 830         } id = { .vlan_id = 0, };
 831
 832         if (!mask)
 833                 mask = &rte_flow_item_vxlan_mask;
 834         if (spec) {
 835                 memcpy(&id.vni[1], spec->vni, 3);
 836                 vxlan.val.tunnel_id = id.vlan_id;
 837                 memcpy(&id.vni[1], mask->vni, 3);
 838                 vxlan.mask.tunnel_id = id.vlan_id;
 839                 /* Remove unwanted bits from values. */
 840                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 841         }
 842         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 843 }
 844
 845 /**
 846  * Convert the @p item into a Verbs specification. This function assumes that
 847  * the input is valid and that there is space to insert the requested item
 848  * into the flow.
 849  *
 850  * @param[in, out] dev_flow
 851  *   Pointer to dev_flow structure.
 852  * @param[in] item
 853  *   Item specification.
 854  * @param[in] item_flags
 855  *   Parsed item flags.
 856  */
 857 static void
 858 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 859                                     const struct rte_flow_item *item,
 860                                     uint64_t item_flags __rte_unused)
 861 {
 862         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 863         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 864         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 865         struct ibv_flow_spec_tunnel vxlan_gpe = {
 866                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 867                 .size = size,
 868         };
 869         union vni {
 870                 uint32_t vlan_id;
 871                 uint8_t vni[4];
 872         } id = { .vlan_id = 0, };
 873
 874         if (!mask)
 875                 mask = &rte_flow_item_vxlan_gpe_mask;
 876         if (spec) {
 877                 memcpy(&id.vni[1], spec->vni, 3);
 878                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 879                 memcpy(&id.vni[1], mask->vni, 3);
 880                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 881                 /* Remove unwanted bits from values. */
 882                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 883         }
 884         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 885 }
 886
 887 /**
 888  * Update the protocol in Verbs IPv4/IPv6 spec.
 889  *
 890  * @param[in, out] attr
 891  *   Pointer to Verbs attributes structure.
 892  * @param[in] search
 893  *   Specification type to search in order to update the IP protocol.
 894  * @param[in] protocol
 895  *   Protocol value to set if none is present in the specification.
 896  */
 897 static void
 898 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 899                                        enum ibv_flow_spec_type search,
 900                                        uint8_t protocol)
 901 {
 902         unsigned int i;
 903         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 904                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 905
 906         if (!attr)
 907                 return;
 908         for (i = 0; i != attr->num_of_specs; ++i) {
 909                 if (hdr->type == search) {
 910                         union {
 911                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 912                                 struct ibv_flow_spec_ipv6 *ipv6;
 913                         } ip;
 914
 915                         switch (search) {
 916                         case IBV_FLOW_SPEC_IPV4_EXT:
 917                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 918                                 if (!ip.ipv4->val.proto) {
 919                                         ip.ipv4->val.proto = protocol;
 920                                         ip.ipv4->mask.proto = 0xff;
 921                                 }
 922                                 break;
 923                         case IBV_FLOW_SPEC_IPV6:
 924                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 925                                 if (!ip.ipv6->val.next_hdr) {
 926                                         ip.ipv6->val.next_hdr = protocol;
 927                                         ip.ipv6->mask.next_hdr = 0xff;
 928                                 }
 929                                 break;
 930                         default:
 931                                 break;
 932                         }
 933                         break;
 934                 }
 935                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 936         }
 937 }
 938
 939 /**
 940  * Convert the @p item into a Verbs specification. This function assumes that
 941  * the input is valid and that there is space to insert the requested item
 942  * into the flow.
 943  *
 944  * @param[in, out] dev_flow
 945  *   Pointer to dev_flow structure.
 946  * @param[in] item
 947  *   Item specification.
 948  * @param[in] item_flags
 949  *   Parsed item flags.
 950  */
 951 static void
 952 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 953                               const struct rte_flow_item *item __rte_unused,
 954                               uint64_t item_flags)
 955 {
 956         struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
 957 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 958         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 959         struct ibv_flow_spec_tunnel tunnel = {
 960                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 961                 .size = size,
 962         };
 963 #else
 964         const struct rte_flow_item_gre *spec = item->spec;
 965         const struct rte_flow_item_gre *mask = item->mask;
 966         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 967         struct ibv_flow_spec_gre tunnel = {
 968                 .type = IBV_FLOW_SPEC_GRE,
 969                 .size = size,
 970         };
 971
 972         if (!mask)
 973                 mask = &rte_flow_item_gre_mask;
 974         if (spec) {
 975                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 976                 tunnel.val.protocol = spec->protocol;
 977                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 978                 tunnel.mask.protocol = mask->protocol;
 979                 /* Remove unwanted bits from values. */
 980                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 981                 tunnel.val.protocol &= tunnel.mask.protocol;
 982                 tunnel.val.key &= tunnel.mask.key;
 983         }
 984 #endif
 985         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 986                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 987                                                        IBV_FLOW_SPEC_IPV4_EXT,
 988                                                        IPPROTO_GRE);
 989         else
 990                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 991                                                        IBV_FLOW_SPEC_IPV6,
 992                                                        IPPROTO_GRE);
 993         flow_verbs_spec_add(verbs, &tunnel, size);
 994 }
 995
 996 /**
 997  * Convert the @p action into a Verbs specification. This function assumes that
 998  * the input is valid and that there is space to insert the requested action
 999  * into the flow. This function also return the action that was added.
1000  *
1001  * @param[in, out] dev_flow
1002  *   Pointer to dev_flow structure.
1003  * @param[in] item
1004  *   Item specification.
1005  * @param[in] item_flags
1006  *   Parsed item flags.
1007  */
1008 static void
1009 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
1010                                const struct rte_flow_item *item __rte_unused,
1011                                uint64_t item_flags __rte_unused)
1012 {
1013 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1014         const struct rte_flow_item_mpls *spec = item->spec;
1015         const struct rte_flow_item_mpls *mask = item->mask;
1016         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
1017         struct ibv_flow_spec_mpls mpls = {
1018                 .type = IBV_FLOW_SPEC_MPLS,
1019                 .size = size,
1020         };
1021
1022         if (!mask)
1023                 mask = &rte_flow_item_mpls_mask;
1024         if (spec) {
1025                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
1026                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
1027                 /* Remove unwanted bits from values.  */
1028                 mpls.val.label &= mpls.mask.label;
1029         }
1030         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
1031 #endif
1032 }
1033
1034 /**
1035  * Convert the @p action into a Verbs specification. This function assumes that
1036  * the input is valid and that there is space to insert the requested action
1037  * into the flow.
1038  *
1039  * @param[in] dev_flow
1040  *   Pointer to mlx5_flow.
1041  * @param[in] action
1042  *   Action configuration.
1043  */
1044 static void
1045 flow_verbs_translate_action_drop
1046         (struct mlx5_flow *dev_flow,
1047          const struct rte_flow_action *action __rte_unused)
1048 {
1049         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1050         struct ibv_flow_spec_action_drop drop = {
1051                         .type = IBV_FLOW_SPEC_ACTION_DROP,
1052                         .size = size,
1053         };
1054
1055         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
1056 }
1057
1058 /**
1059  * Convert the @p action into a Verbs specification. This function assumes that
1060  * the input is valid and that there is space to insert the requested action
1061  * into the flow.
1062  *
1063  * @param[in] rss_desc
1064  *   Pointer to mlx5_flow_rss_desc.
1065  * @param[in] action
1066  *   Action configuration.
1067  */
1068 static void
1069 flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc,
1070                                   const struct rte_flow_action *action)
1071 {
1072         const struct rte_flow_action_queue *queue = action->conf;
1073
1074         rss_desc->queue[0] = queue->index;
1075         rss_desc->queue_num = 1;
1076 }
1077
1078 /**
1079  * Convert the @p action into a Verbs specification. This function assumes that
1080  * the input is valid and that there is space to insert the requested action
1081  * into the flow.
1082  *
1083  * @param[in] rss_desc
1084  *   Pointer to mlx5_flow_rss_desc.
1085  * @param[in] action
1086  *   Action configuration.
1087  */
1088 static void
1089 flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc,
1090                                 const struct rte_flow_action *action)
1091 {
1092         const struct rte_flow_action_rss *rss = action->conf;
1093         const uint8_t *rss_key;
1094
1095         memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t));
1096         rss_desc->queue_num = rss->queue_num;
1097         /* NULL RSS key indicates default RSS key. */
1098         rss_key = !rss->key ? rss_hash_default_key : rss->key;
1099         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
1100         /*
1101          * rss->level and rss.types should be set in advance when expanding
1102          * items for RSS.
1103          */
1104 }
1105
1106 /**
1107  * Convert the @p action into a Verbs specification. This function assumes that
1108  * the input is valid and that there is space to insert the requested action
1109  * into the flow.
1110  *
1111  * @param[in] dev_flow
1112  *   Pointer to mlx5_flow.
1113  * @param[in] action
1114  *   Action configuration.
1115  */
1116 static void
1117 flow_verbs_translate_action_flag
1118         (struct mlx5_flow *dev_flow,
1119          const struct rte_flow_action *action __rte_unused)
1120 {
1121         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1122         struct ibv_flow_spec_action_tag tag = {
1123                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1124                 .size = size,
1125                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
1126         };
1127
1128         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1129 }
1130
1131 /**
1132  * Convert the @p action into a Verbs specification. This function assumes that
1133  * the input is valid and that there is space to insert the requested action
1134  * into the flow.
1135  *
1136  * @param[in] dev_flow
1137  *   Pointer to mlx5_flow.
1138  * @param[in] action
1139  *   Action configuration.
1140  */
1141 static void
1142 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
1143                                  const struct rte_flow_action *action)
1144 {
1145         const struct rte_flow_action_mark *mark = action->conf;
1146         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1147         struct ibv_flow_spec_action_tag tag = {
1148                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1149                 .size = size,
1150                 .tag_id = mlx5_flow_mark_set(mark->id),
1151         };
1152
1153         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
1154 }
1155
1156 /**
1157  * Convert the @p action into a Verbs specification. This function assumes that
1158  * the input is valid and that there is space to insert the requested action
1159  * into the flow.
1160  *
1161  * @param[in] dev
1162  *   Pointer to the Ethernet device structure.
1163  * @param[in] action
1164  *   Action configuration.
1165  * @param[in] dev_flow
1166  *   Pointer to mlx5_flow.
1167  * @param[out] error
1168  *   Pointer to error structure.
1169  *
1170  * @return
1171  *   0 On success else a negative errno value is returned and rte_errno is set.
1172  */
1173 static int
1174 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
1175                                   const struct rte_flow_action *action,
1176                                   struct rte_eth_dev *dev,
1177                                   struct rte_flow_error *error)
1178 {
1179         const struct rte_flow_action_count *count = action->conf;
1180         struct rte_flow *flow = dev_flow->flow;
1181 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1182         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1183         struct mlx5_flow_counter_pool *pool;
1184         struct mlx5_flow_counter *cnt = NULL;
1185         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1186         struct ibv_flow_spec_counter_action counter = {
1187                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1188                 .size = size,
1189         };
1190 #endif
1191
1192         if (!flow->counter) {
1193                 flow->counter = flow_verbs_counter_new(dev, count->shared,
1194                                                        count->id);
1195                 if (!flow->counter)
1196                         return rte_flow_error_set(error, rte_errno,
1197                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1198                                                   action,
1199                                                   "cannot get counter"
1200                                                   " context.");
1201         }
1202 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1203         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1204         counter.counter_set_handle =
1205                 ((struct ibv_counter_set *)cnt->dcs_when_active)->handle;
1206         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1207 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1208         cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool);
1209         counter.counters = (struct ibv_counters *)cnt->dcs_when_active;
1210         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1211 #endif
1212         return 0;
1213 }
1214
1215 /**
1216  * Internal validation function. For validating both actions and items.
1217  *
1218  * @param[in] dev
1219  *   Pointer to the Ethernet device structure.
1220  * @param[in] attr
1221  *   Pointer to the flow attributes.
1222  * @param[in] items
1223  *   Pointer to the list of items.
1224  * @param[in] actions
1225  *   Pointer to the list of actions.
1226  * @param[in] external
1227  *   This flow rule is created by request external to PMD.
1228  * @param[in] hairpin
1229  *   Number of hairpin TX actions, 0 means classic flow.
1230  * @param[out] error
1231  *   Pointer to the error structure.
1232  *
1233  * @return
1234  *   0 on success, a negative errno value otherwise and rte_errno is set.
1235  */
1236 static int
1237 flow_verbs_validate(struct rte_eth_dev *dev,
1238                     const struct rte_flow_attr *attr,
1239                     const struct rte_flow_item items[],
1240                     const struct rte_flow_action actions[],
1241                     bool external __rte_unused,
1242                     int hairpin __rte_unused,
1243                     struct rte_flow_error *error)
1244 {
1245         int ret;
1246         uint64_t action_flags = 0;
1247         uint64_t item_flags = 0;
1248         uint64_t last_item = 0;
1249         uint8_t next_protocol = 0xff;
1250         uint16_t ether_type = 0;
1251
1252         if (items == NULL)
1253                 return -1;
1254         ret = mlx5_flow_validate_attributes(dev, attr, error);
1255         if (ret < 0)
1256                 return ret;
1257         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1258                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1259                 int ret = 0;
1260
1261                 switch (items->type) {
1262                 case RTE_FLOW_ITEM_TYPE_VOID:
1263                         break;
1264                 case RTE_FLOW_ITEM_TYPE_ETH:
1265                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1266                                                           error);
1267                         if (ret < 0)
1268                                 return ret;
1269                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1270                                              MLX5_FLOW_LAYER_OUTER_L2;
1271                         if (items->mask != NULL && items->spec != NULL) {
1272                                 ether_type =
1273                                         ((const struct rte_flow_item_eth *)
1274                                          items->spec)->type;
1275                                 ether_type &=
1276                                         ((const struct rte_flow_item_eth *)
1277                                          items->mask)->type;
1278                                 ether_type = rte_be_to_cpu_16(ether_type);
1279                         } else {
1280                                 ether_type = 0;
1281                         }
1282                         break;
1283                 case RTE_FLOW_ITEM_TYPE_VLAN:
1284                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1285                                                            dev, error);
1286                         if (ret < 0)
1287                                 return ret;
1288                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1289                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1290                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1291                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1292                         if (items->mask != NULL && items->spec != NULL) {
1293                                 ether_type =
1294                                         ((const struct rte_flow_item_vlan *)
1295                                          items->spec)->inner_type;
1296                                 ether_type &=
1297                                         ((const struct rte_flow_item_vlan *)
1298                                          items->mask)->inner_type;
1299                                 ether_type = rte_be_to_cpu_16(ether_type);
1300                         } else {
1301                                 ether_type = 0;
1302                         }
1303                         break;
1304                 case RTE_FLOW_ITEM_TYPE_IPV4:
1305                         ret = mlx5_flow_validate_item_ipv4
1306                                                 (items, item_flags,
1307                                                  last_item, ether_type, NULL,
1308                                                  MLX5_ITEM_RANGE_NOT_ACCEPTED,
1309                                                  error);
1310                         if (ret < 0)
1311                                 return ret;
1312                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1313                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1314                         if (items->mask != NULL &&
1315                             ((const struct rte_flow_item_ipv4 *)
1316                              items->mask)->hdr.next_proto_id) {
1317                                 next_protocol =
1318                                         ((const struct rte_flow_item_ipv4 *)
1319                                          (items->spec))->hdr.next_proto_id;
1320                                 next_protocol &=
1321                                         ((const struct rte_flow_item_ipv4 *)
1322                                          (items->mask))->hdr.next_proto_id;
1323                         } else {
1324                                 /* Reset for inner layer. */
1325                                 next_protocol = 0xff;
1326                         }
1327                         break;
1328                 case RTE_FLOW_ITEM_TYPE_IPV6:
1329                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1330                                                            last_item,
1331                                                            ether_type, NULL,
1332                                                            error);
1333                         if (ret < 0)
1334                                 return ret;
1335                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1336                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1337                         if (items->mask != NULL &&
1338                             ((const struct rte_flow_item_ipv6 *)
1339                              items->mask)->hdr.proto) {
1340                                 next_protocol =
1341                                         ((const struct rte_flow_item_ipv6 *)
1342                                          items->spec)->hdr.proto;
1343                                 next_protocol &=
1344                                         ((const struct rte_flow_item_ipv6 *)
1345                                          items->mask)->hdr.proto;
1346                         } else {
1347                                 /* Reset for inner layer. */
1348                                 next_protocol = 0xff;
1349                         }
1350                         break;
1351                 case RTE_FLOW_ITEM_TYPE_UDP:
1352                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1353                                                           next_protocol,
1354                                                           error);
1355                         if (ret < 0)
1356                                 return ret;
1357                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1358                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1359                         break;
1360                 case RTE_FLOW_ITEM_TYPE_TCP:
1361                         ret = mlx5_flow_validate_item_tcp
1362                                                 (items, item_flags,
1363                                                  next_protocol,
1364                                                  &rte_flow_item_tcp_mask,
1365                                                  error);
1366                         if (ret < 0)
1367                                 return ret;
1368                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1369                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1370                         break;
1371                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1372                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1373                                                             error);
1374                         if (ret < 0)
1375                                 return ret;
1376                         last_item = MLX5_FLOW_LAYER_VXLAN;
1377                         break;
1378                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1379                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1380                                                                 item_flags,
1381                                                                 dev, error);
1382                         if (ret < 0)
1383                                 return ret;
1384                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1385                         break;
1386                 case RTE_FLOW_ITEM_TYPE_GRE:
1387                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1388                                                           next_protocol, error);
1389                         if (ret < 0)
1390                                 return ret;
1391                         last_item = MLX5_FLOW_LAYER_GRE;
1392                         break;
1393                 case RTE_FLOW_ITEM_TYPE_MPLS:
1394                         ret = mlx5_flow_validate_item_mpls(dev, items,
1395                                                            item_flags,
1396                                                            last_item, error);
1397                         if (ret < 0)
1398                                 return ret;
1399                         last_item = MLX5_FLOW_LAYER_MPLS;
1400                         break;
1401                 default:
1402                         return rte_flow_error_set(error, ENOTSUP,
1403                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1404                                                   NULL, "item not supported");
1405                 }
1406                 item_flags |= last_item;
1407         }
1408         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1409                 switch (actions->type) {
1410                 case RTE_FLOW_ACTION_TYPE_VOID:
1411                         break;
1412                 case RTE_FLOW_ACTION_TYPE_FLAG:
1413                         ret = mlx5_flow_validate_action_flag(action_flags,
1414                                                              attr,
1415                                                              error);
1416                         if (ret < 0)
1417                                 return ret;
1418                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1419                         break;
1420                 case RTE_FLOW_ACTION_TYPE_MARK:
1421                         ret = mlx5_flow_validate_action_mark(actions,
1422                                                              action_flags,
1423                                                              attr,
1424                                                              error);
1425                         if (ret < 0)
1426                                 return ret;
1427                         action_flags |= MLX5_FLOW_ACTION_MARK;
1428                         break;
1429                 case RTE_FLOW_ACTION_TYPE_DROP:
1430                         ret = mlx5_flow_validate_action_drop(action_flags,
1431                                                              attr,
1432                                                              error);
1433                         if (ret < 0)
1434                                 return ret;
1435                         action_flags |= MLX5_FLOW_ACTION_DROP;
1436                         break;
1437                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1438                         ret = mlx5_flow_validate_action_queue(actions,
1439                                                               action_flags, dev,
1440                                                               attr,
1441                                                               error);
1442                         if (ret < 0)
1443                                 return ret;
1444                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1445                         break;
1446                 case RTE_FLOW_ACTION_TYPE_RSS:
1447                         ret = mlx5_flow_validate_action_rss(actions,
1448                                                             action_flags, dev,
1449                                                             attr, item_flags,
1450                                                             error);
1451                         if (ret < 0)
1452                                 return ret;
1453                         action_flags |= MLX5_FLOW_ACTION_RSS;
1454                         break;
1455                 case RTE_FLOW_ACTION_TYPE_COUNT:
1456                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1457                         if (ret < 0)
1458                                 return ret;
1459                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1460                         break;
1461                 default:
1462                         return rte_flow_error_set(error, ENOTSUP,
1463                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1464                                                   actions,
1465                                                   "action not supported");
1466                 }
1467         }
1468         /*
1469          * Validate the drop action mutual exclusion with other actions.
1470          * Drop action is mutually-exclusive with any other action, except for
1471          * Count action.
1472          */
1473         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1474             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1475                 return rte_flow_error_set(error, EINVAL,
1476                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1477                                           "Drop action is mutually-exclusive "
1478                                           "with any other action, except for "
1479                                           "Count action");
1480         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1481                 return rte_flow_error_set(error, EINVAL,
1482                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1483                                           "no fate action is found");
1484         return 0;
1485 }
1486
1487 /**
1488  * Calculate the required bytes that are needed for the action part of the verbs
1489  * flow.
1490  *
1491  * @param[in] actions
1492  *   Pointer to the list of actions.
1493  *
1494  * @return
1495  *   The size of the memory needed for all actions.
1496  */
1497 static int
1498 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1499 {
1500         int size = 0;
1501
1502         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1503                 switch (actions->type) {
1504                 case RTE_FLOW_ACTION_TYPE_VOID:
1505                         break;
1506                 case RTE_FLOW_ACTION_TYPE_FLAG:
1507                         size += sizeof(struct ibv_flow_spec_action_tag);
1508                         break;
1509                 case RTE_FLOW_ACTION_TYPE_MARK:
1510                         size += sizeof(struct ibv_flow_spec_action_tag);
1511                         break;
1512                 case RTE_FLOW_ACTION_TYPE_DROP:
1513                         size += sizeof(struct ibv_flow_spec_action_drop);
1514                         break;
1515                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1516                         break;
1517                 case RTE_FLOW_ACTION_TYPE_RSS:
1518                         break;
1519                 case RTE_FLOW_ACTION_TYPE_COUNT:
1520 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1521         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1522                         size += sizeof(struct ibv_flow_spec_counter_action);
1523 #endif
1524                         break;
1525                 default:
1526                         break;
1527                 }
1528         }
1529         return size;
1530 }
1531
1532 /**
1533  * Calculate the required bytes that are needed for the item part of the verbs
1534  * flow.
1535  *
1536  * @param[in] items
1537  *   Pointer to the list of items.
1538  *
1539  * @return
1540  *   The size of the memory needed for all items.
1541  */
1542 static int
1543 flow_verbs_get_items_size(const struct rte_flow_item items[])
1544 {
1545         int size = 0;
1546
1547         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1548                 switch (items->type) {
1549                 case RTE_FLOW_ITEM_TYPE_VOID:
1550                         break;
1551                 case RTE_FLOW_ITEM_TYPE_ETH:
1552                         size += sizeof(struct ibv_flow_spec_eth);
1553                         break;
1554                 case RTE_FLOW_ITEM_TYPE_VLAN:
1555                         size += sizeof(struct ibv_flow_spec_eth);
1556                         break;
1557                 case RTE_FLOW_ITEM_TYPE_IPV4:
1558                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1559                         break;
1560                 case RTE_FLOW_ITEM_TYPE_IPV6:
1561                         size += sizeof(struct ibv_flow_spec_ipv6);
1562                         break;
1563                 case RTE_FLOW_ITEM_TYPE_UDP:
1564                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1565                         break;
1566                 case RTE_FLOW_ITEM_TYPE_TCP:
1567                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1568                         break;
1569                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1570                         size += sizeof(struct ibv_flow_spec_tunnel);
1571                         break;
1572                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1573                         size += sizeof(struct ibv_flow_spec_tunnel);
1574                         break;
1575 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1576                 case RTE_FLOW_ITEM_TYPE_GRE:
1577                         size += sizeof(struct ibv_flow_spec_gre);
1578                         break;
1579                 case RTE_FLOW_ITEM_TYPE_MPLS:
1580                         size += sizeof(struct ibv_flow_spec_mpls);
1581                         break;
1582 #else
1583                 case RTE_FLOW_ITEM_TYPE_GRE:
1584                         size += sizeof(struct ibv_flow_spec_tunnel);
1585                         break;
1586 #endif
1587                 default:
1588                         break;
1589                 }
1590         }
1591         return size;
1592 }
1593
1594 /**
1595  * Internal preparation function. Allocate mlx5_flow with the required size.
1596  * The required size is calculate based on the actions and items. This function
1597  * also returns the detected actions and items for later use.
1598  *
1599  * @param[in] dev
1600  *   Pointer to Ethernet device.
1601  * @param[in] attr
1602  *   Pointer to the flow attributes.
1603  * @param[in] items
1604  *   Pointer to the list of items.
1605  * @param[in] actions
1606  *   Pointer to the list of actions.
1607  * @param[out] error
1608  *   Pointer to the error structure.
1609  *
1610  * @return
1611  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1612  *   is set.
1613  */
1614 static struct mlx5_flow *
1615 flow_verbs_prepare(struct rte_eth_dev *dev,
1616                    const struct rte_flow_attr *attr __rte_unused,
1617                    const struct rte_flow_item items[],
1618                    const struct rte_flow_action actions[],
1619                    struct rte_flow_error *error)
1620 {
1621         size_t size = 0;
1622         uint32_t handle_idx = 0;
1623         struct mlx5_flow *dev_flow;
1624         struct mlx5_flow_handle *dev_handle;
1625         struct mlx5_priv *priv = dev->data->dev_private;
1626
1627         size += flow_verbs_get_actions_size(actions);
1628         size += flow_verbs_get_items_size(items);
1629         if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1630                 rte_flow_error_set(error, E2BIG,
1631                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1632                                    "Verbs spec/action size too large");
1633                 return NULL;
1634         }
1635         /* In case of corrupting the memory. */
1636         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1637                 rte_flow_error_set(error, ENOSPC,
1638                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1639                                    "not free temporary device flow");
1640                 return NULL;
1641         }
1642         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1643                                    &handle_idx);
1644         if (!dev_handle) {
1645                 rte_flow_error_set(error, ENOMEM,
1646                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1647                                    "not enough memory to create flow handle");
1648                 return NULL;
1649         }
1650         /* No multi-thread supporting. */
1651         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
1652         dev_flow->handle = dev_handle;
1653         dev_flow->handle_idx = handle_idx;
1654         /* Memcpy is used, only size needs to be cleared to 0. */
1655         dev_flow->verbs.size = 0;
1656         dev_flow->verbs.attr.num_of_specs = 0;
1657         dev_flow->ingress = attr->ingress;
1658         dev_flow->hash_fields = 0;
1659         /* Need to set transfer attribute: not supported in Verbs mode. */
1660         return dev_flow;
1661 }
1662
1663 /**
1664  * Fill the flow with verb spec.
1665  *
1666  * @param[in] dev
1667  *   Pointer to Ethernet device.
1668  * @param[in, out] dev_flow
1669  *   Pointer to the mlx5 flow.
1670  * @param[in] attr
1671  *   Pointer to the flow attributes.
1672  * @param[in] items
1673  *   Pointer to the list of items.
1674  * @param[in] actions
1675  *   Pointer to the list of actions.
1676  * @param[out] error
1677  *   Pointer to the error structure.
1678  *
1679  * @return
1680  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1681  */
1682 static int
1683 flow_verbs_translate(struct rte_eth_dev *dev,
1684                      struct mlx5_flow *dev_flow,
1685                      const struct rte_flow_attr *attr,
1686                      const struct rte_flow_item items[],
1687                      const struct rte_flow_action actions[],
1688                      struct rte_flow_error *error)
1689 {
1690         uint64_t item_flags = 0;
1691         uint64_t action_flags = 0;
1692         uint64_t priority = attr->priority;
1693         uint32_t subpriority = 0;
1694         struct mlx5_priv *priv = dev->data->dev_private;
1695         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
1696                                               priv->rss_desc)
1697                                               [!!priv->flow_nested_idx];
1698
1699         if (priority == MLX5_FLOW_PRIO_RSVD)
1700                 priority = priv->config.flow_prio - 1;
1701         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1702                 int ret;
1703
1704                 switch (actions->type) {
1705                 case RTE_FLOW_ACTION_TYPE_VOID:
1706                         break;
1707                 case RTE_FLOW_ACTION_TYPE_FLAG:
1708                         flow_verbs_translate_action_flag(dev_flow, actions);
1709                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1710                         dev_flow->handle->mark = 1;
1711                         break;
1712                 case RTE_FLOW_ACTION_TYPE_MARK:
1713                         flow_verbs_translate_action_mark(dev_flow, actions);
1714                         action_flags |= MLX5_FLOW_ACTION_MARK;
1715                         dev_flow->handle->mark = 1;
1716                         break;
1717                 case RTE_FLOW_ACTION_TYPE_DROP:
1718                         flow_verbs_translate_action_drop(dev_flow, actions);
1719                         action_flags |= MLX5_FLOW_ACTION_DROP;
1720                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
1721                         break;
1722                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1723                         flow_verbs_translate_action_queue(rss_desc, actions);
1724                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1725                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1726                         break;
1727                 case RTE_FLOW_ACTION_TYPE_RSS:
1728                         flow_verbs_translate_action_rss(rss_desc, actions);
1729                         action_flags |= MLX5_FLOW_ACTION_RSS;
1730                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
1731                         break;
1732                 case RTE_FLOW_ACTION_TYPE_COUNT:
1733                         ret = flow_verbs_translate_action_count(dev_flow,
1734                                                                 actions,
1735                                                                 dev, error);
1736                         if (ret < 0)
1737                                 return ret;
1738                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1739                         break;
1740                 default:
1741                         return rte_flow_error_set(error, ENOTSUP,
1742                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1743                                                   actions,
1744                                                   "action not supported");
1745                 }
1746         }
1747         dev_flow->act_flags = action_flags;
1748         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1749                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1750
1751                 switch (items->type) {
1752                 case RTE_FLOW_ITEM_TYPE_VOID:
1753                         break;
1754                 case RTE_FLOW_ITEM_TYPE_ETH:
1755                         flow_verbs_translate_item_eth(dev_flow, items,
1756                                                       item_flags);
1757                         subpriority = MLX5_PRIORITY_MAP_L2;
1758                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1759                                                MLX5_FLOW_LAYER_OUTER_L2;
1760                         break;
1761                 case RTE_FLOW_ITEM_TYPE_VLAN:
1762                         flow_verbs_translate_item_vlan(dev_flow, items,
1763                                                        item_flags);
1764                         subpriority = MLX5_PRIORITY_MAP_L2;
1765                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1766                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1767                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1768                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1769                         break;
1770                 case RTE_FLOW_ITEM_TYPE_IPV4:
1771                         flow_verbs_translate_item_ipv4(dev_flow, items,
1772                                                        item_flags);
1773                         subpriority = MLX5_PRIORITY_MAP_L3;
1774                         dev_flow->hash_fields |=
1775                                 mlx5_flow_hashfields_adjust
1776                                         (rss_desc, tunnel,
1777                                          MLX5_IPV4_LAYER_TYPES,
1778                                          MLX5_IPV4_IBV_RX_HASH);
1779                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1780                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1781                         break;
1782                 case RTE_FLOW_ITEM_TYPE_IPV6:
1783                         flow_verbs_translate_item_ipv6(dev_flow, items,
1784                                                        item_flags);
1785                         subpriority = MLX5_PRIORITY_MAP_L3;
1786                         dev_flow->hash_fields |=
1787                                 mlx5_flow_hashfields_adjust
1788                                         (rss_desc, tunnel,
1789                                          MLX5_IPV6_LAYER_TYPES,
1790                                          MLX5_IPV6_IBV_RX_HASH);
1791                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1792                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1793                         break;
1794                 case RTE_FLOW_ITEM_TYPE_TCP:
1795                         flow_verbs_translate_item_tcp(dev_flow, items,
1796                                                       item_flags);
1797                         subpriority = MLX5_PRIORITY_MAP_L4;
1798                         dev_flow->hash_fields |=
1799                                 mlx5_flow_hashfields_adjust
1800                                         (rss_desc, tunnel, ETH_RSS_TCP,
1801                                          (IBV_RX_HASH_SRC_PORT_TCP |
1802                                           IBV_RX_HASH_DST_PORT_TCP));
1803                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1804                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1805                         break;
1806                 case RTE_FLOW_ITEM_TYPE_UDP:
1807                         flow_verbs_translate_item_udp(dev_flow, items,
1808                                                       item_flags);
1809                         subpriority = MLX5_PRIORITY_MAP_L4;
1810                         dev_flow->hash_fields |=
1811                                 mlx5_flow_hashfields_adjust
1812                                         (rss_desc, tunnel, ETH_RSS_UDP,
1813                                          (IBV_RX_HASH_SRC_PORT_UDP |
1814                                           IBV_RX_HASH_DST_PORT_UDP));
1815                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1816                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1817                         break;
1818                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1819                         flow_verbs_translate_item_vxlan(dev_flow, items,
1820                                                         item_flags);
1821                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1822                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1823                         break;
1824                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1825                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1826                                                             item_flags);
1827                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1828                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1829                         break;
1830                 case RTE_FLOW_ITEM_TYPE_GRE:
1831                         flow_verbs_translate_item_gre(dev_flow, items,
1832                                                       item_flags);
1833                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1834                         item_flags |= MLX5_FLOW_LAYER_GRE;
1835                         break;
1836                 case RTE_FLOW_ITEM_TYPE_MPLS:
1837                         flow_verbs_translate_item_mpls(dev_flow, items,
1838                                                        item_flags);
1839                         subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc);
1840                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1841                         break;
1842                 default:
1843                         return rte_flow_error_set(error, ENOTSUP,
1844                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1845                                                   NULL,
1846                                                   "item not supported");
1847                 }
1848         }
1849         dev_flow->handle->layers = item_flags;
1850         /* Other members of attr will be ignored. */
1851         dev_flow->verbs.attr.priority =
1852                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1853         dev_flow->verbs.attr.port = (uint8_t)priv->dev_port;
1854         return 0;
1855 }
1856
1857 /**
1858  * Remove the flow from the NIC but keeps it in memory.
1859  *
1860  * @param[in] dev
1861  *   Pointer to the Ethernet device structure.
1862  * @param[in, out] flow
1863  *   Pointer to flow structure.
1864  */
1865 static void
1866 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1867 {
1868         struct mlx5_priv *priv = dev->data->dev_private;
1869         struct mlx5_flow_handle *handle;
1870         uint32_t handle_idx;
1871
1872         if (!flow)
1873                 return;
1874         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
1875                        handle_idx, handle, next) {
1876                 if (handle->drv_flow) {
1877                         claim_zero(mlx5_glue->destroy_flow(handle->drv_flow));
1878                         handle->drv_flow = NULL;
1879                 }
1880                 /* hrxq is union, don't touch it only the flag is set. */
1881                 if (handle->rix_hrxq) {
1882                         if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1883                                 mlx5_drop_action_destroy(dev);
1884                                 handle->rix_hrxq = 0;
1885                         } else if (handle->fate_action ==
1886                                    MLX5_FLOW_FATE_QUEUE) {
1887                                 mlx5_hrxq_release(dev, handle->rix_hrxq);
1888                                 handle->rix_hrxq = 0;
1889                         }
1890                 }
1891                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1892                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1893         }
1894 }
1895
1896 /**
1897  * Remove the flow from the NIC and the memory.
1898  *
1899  * @param[in] dev
1900  *   Pointer to the Ethernet device structure.
1901  * @param[in, out] flow
1902  *   Pointer to flow structure.
1903  */
1904 static void
1905 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1906 {
1907         struct mlx5_priv *priv = dev->data->dev_private;
1908         struct mlx5_flow_handle *handle;
1909
1910         if (!flow)
1911                 return;
1912         flow_verbs_remove(dev, flow);
1913         while (flow->dev_handles) {
1914                 uint32_t tmp_idx = flow->dev_handles;
1915
1916                 handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1917                                    tmp_idx);
1918                 if (!handle)
1919                         return;
1920                 flow->dev_handles = handle->next.next;
1921                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
1922                            tmp_idx);
1923         }
1924         if (flow->counter) {
1925                 flow_verbs_counter_release(dev, flow->counter);
1926                 flow->counter = 0;
1927         }
1928 }
1929
1930 /**
1931  * Apply the flow to the NIC.
1932  *
1933  * @param[in] dev
1934  *   Pointer to the Ethernet device structure.
1935  * @param[in, out] flow
1936  *   Pointer to flow structure.
1937  * @param[out] error
1938  *   Pointer to error structure.
1939  *
1940  * @return
1941  *   0 on success, a negative errno value otherwise and rte_errno is set.
1942  */
1943 static int
1944 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1945                  struct rte_flow_error *error)
1946 {
1947         struct mlx5_priv *priv = dev->data->dev_private;
1948         struct mlx5_flow_handle *handle;
1949         struct mlx5_flow *dev_flow;
1950         struct mlx5_hrxq *hrxq;
1951         uint32_t dev_handles;
1952         int err;
1953         int idx;
1954
1955         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
1956                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
1957                 handle = dev_flow->handle;
1958                 if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
1959                         hrxq = mlx5_drop_action_create(dev);
1960                         if (!hrxq) {
1961                                 rte_flow_error_set
1962                                         (error, errno,
1963                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1964                                          "cannot get drop hash queue");
1965                                 goto error;
1966                         }
1967                 } else {
1968                         uint32_t hrxq_idx;
1969                         struct mlx5_flow_rss_desc *rss_desc =
1970                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
1971                                 [!!priv->flow_nested_idx];
1972
1973                         MLX5_ASSERT(rss_desc->queue_num);
1974                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
1975                                                  MLX5_RSS_HASH_KEY_LEN,
1976                                                  dev_flow->hash_fields,
1977                                                  rss_desc->queue,
1978                                                  rss_desc->queue_num);
1979                         if (!hrxq_idx)
1980                                 hrxq_idx = mlx5_hrxq_new
1981                                                 (dev, rss_desc->key,
1982                                                  MLX5_RSS_HASH_KEY_LEN,
1983                                                  dev_flow->hash_fields,
1984                                                  rss_desc->queue,
1985                                                  rss_desc->queue_num,
1986                                                  !!(handle->layers &
1987                                                  MLX5_FLOW_LAYER_TUNNEL),
1988                                                  false);
1989                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
1990                                               hrxq_idx);
1991                         if (!hrxq) {
1992                                 rte_flow_error_set
1993                                         (error, rte_errno,
1994                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1995                                          "cannot get hash queue");
1996                                 goto error;
1997                         }
1998                         handle->rix_hrxq = hrxq_idx;
1999                 }
2000                 MLX5_ASSERT(hrxq);
2001                 handle->drv_flow = mlx5_glue->create_flow
2002                                         (hrxq->qp, &dev_flow->verbs.attr);
2003                 if (!handle->drv_flow) {
2004                         rte_flow_error_set(error, errno,
2005                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2006                                            NULL,
2007                                            "hardware refuses to create flow");
2008                         goto error;
2009                 }
2010                 if (priv->vmwa_context &&
2011                     handle->vf_vlan.tag && !handle->vf_vlan.created) {
2012                         /*
2013                          * The rule contains the VLAN pattern.
2014                          * For VF we are going to create VLAN
2015                          * interface to make hypervisor set correct
2016                          * e-Switch vport context.
2017                          */
2018                         mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
2019                 }
2020         }
2021         return 0;
2022 error:
2023         err = rte_errno; /* Save rte_errno before cleanup. */
2024         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
2025                        dev_handles, handle, next) {
2026                 /* hrxq is union, don't touch it only the flag is set. */
2027                 if (handle->rix_hrxq) {
2028                         if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
2029                                 mlx5_drop_action_destroy(dev);
2030                                 handle->rix_hrxq = 0;
2031                         } else if (handle->fate_action ==
2032                                    MLX5_FLOW_FATE_QUEUE) {
2033                                 mlx5_hrxq_release(dev, handle->rix_hrxq);
2034                                 handle->rix_hrxq = 0;
2035                         }
2036                 }
2037                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
2038                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
2039         }
2040         rte_errno = err; /* Restore rte_errno. */
2041         return -rte_errno;
2042 }
2043
2044 /**
2045  * Query a flow.
2046  *
2047  * @see rte_flow_query()
2048  * @see rte_flow_ops
2049  */
2050 static int
2051 flow_verbs_query(struct rte_eth_dev *dev,
2052                  struct rte_flow *flow,
2053                  const struct rte_flow_action *actions,
2054                  void *data,
2055                  struct rte_flow_error *error)
2056 {
2057         int ret = -EINVAL;
2058
2059         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2060                 switch (actions->type) {
2061                 case RTE_FLOW_ACTION_TYPE_VOID:
2062                         break;
2063                 case RTE_FLOW_ACTION_TYPE_COUNT:
2064                         ret = flow_verbs_counter_query(dev, flow, data, error);
2065                         break;
2066                 default:
2067                         return rte_flow_error_set(error, ENOTSUP,
2068                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2069                                                   actions,
2070                                                   "action not supported");
2071                 }
2072         }
2073         return ret;
2074 }
2075
2076 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
2077         .validate = flow_verbs_validate,
2078         .prepare = flow_verbs_prepare,
2079         .translate = flow_verbs_translate,
2080         .apply = flow_verbs_apply,
2081         .remove = flow_verbs_remove,
2082         .destroy = flow_verbs_destroy,
2083         .query = flow_verbs_query,
2084 };