drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28
  29 #include "mlx5.h"
  30 #include "mlx5_defs.h"
  31 #include "mlx5_flow.h"
  32 #include "mlx5_glue.h"
  33 #include "mlx5_prm.h"
  34 #include "mlx5_rxtx.h"
  35
  36 #define VERBS_SPEC_INNER(item_flags) \
  37         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  38
  39 /**
  40  * Create Verbs flow counter with Verbs library.
  41  *
  42  * @param[in] dev
  43  *   Pointer to the Ethernet device structure.
  44  * @param[in, out] counter
  45  *   mlx5 flow counter object, contains the counter id,
  46  *   handle of created Verbs flow counter is returned
  47  *   in cs field (if counters are supported).
  48  *
  49  * @return
  50  *   0 On success else a negative errno value is returned
  51  *   and rte_errno is set.
  52  */
  53 static int
  54 flow_verbs_counter_create(struct rte_eth_dev *dev,
  55                           struct mlx5_flow_counter *counter)
  56 {
  57 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  58         struct mlx5_priv *priv = dev->data->dev_private;
  59         struct ibv_context *ctx = priv->sh->ctx;
  60         struct ibv_counter_set_init_attr init = {
  61                          .counter_set_id = counter->id};
  62
  63         counter->cs = mlx5_glue->create_counter_set(ctx, &init);
  64         if (!counter->cs) {
  65                 rte_errno = ENOTSUP;
  66                 return -ENOTSUP;
  67         }
  68         return 0;
  69 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
  70         struct mlx5_priv *priv = dev->data->dev_private;
  71         struct ibv_context *ctx = priv->sh->ctx;
  72         struct ibv_counters_init_attr init = {0};
  73         struct ibv_counter_attach_attr attach;
  74         int ret;
  75
  76         memset(&attach, 0, sizeof(attach));
  77         counter->cs = mlx5_glue->create_counters(ctx, &init);
  78         if (!counter->cs) {
  79                 rte_errno = ENOTSUP;
  80                 return -ENOTSUP;
  81         }
  82         attach.counter_desc = IBV_COUNTER_PACKETS;
  83         attach.index = 0;
  84         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
  85         if (!ret) {
  86                 attach.counter_desc = IBV_COUNTER_BYTES;
  87                 attach.index = 1;
  88                 ret = mlx5_glue->attach_counters
  89                                         (counter->cs, &attach, NULL);
  90         }
  91         if (ret) {
  92                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
  93                 counter->cs = NULL;
  94                 rte_errno = ret;
  95                 return -ret;
  96         }
  97         return 0;
  98 #else
  99         (void)dev;
 100         (void)counter;
 101         rte_errno = ENOTSUP;
 102         return -ENOTSUP;
 103 #endif
 104 }
 105
 106 /**
 107  * Get a flow counter.
 108  *
 109  * @param[in] dev
 110  *   Pointer to the Ethernet device structure.
 111  * @param[in] shared
 112  *   Indicate if this counter is shared with other flows.
 113  * @param[in] id
 114  *   Counter identifier.
 115  *
 116  * @return
 117  *   A pointer to the counter, NULL otherwise and rte_errno is set.
 118  */
 119 static struct mlx5_flow_counter *
 120 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 121 {
 122         struct mlx5_priv *priv = dev->data->dev_private;
 123         struct mlx5_flow_counter *cnt;
 124         int ret;
 125
 126         if (shared) {
 127                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
 128                         if (cnt->shared && cnt->id == id) {
 129                                 cnt->ref_cnt++;
 130                                 return cnt;
 131                         }
 132                 }
 133         }
 134         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 135         if (!cnt) {
 136                 rte_errno = ENOMEM;
 137                 return NULL;
 138         }
 139         cnt->id = id;
 140         cnt->shared = shared;
 141         cnt->ref_cnt = 1;
 142         cnt->hits = 0;
 143         cnt->bytes = 0;
 144         /* Create counter with Verbs. */
 145         ret = flow_verbs_counter_create(dev, cnt);
 146         if (!ret) {
 147                 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
 148                 return cnt;
 149         }
 150         /* Some error occurred in Verbs library. */
 151         rte_free(cnt);
 152         rte_errno = -ret;
 153         return NULL;
 154 }
 155
 156 /**
 157  * Release a flow counter.
 158  *
 159  * @param[in] dev
 160  *   Pointer to the Ethernet device structure.
 161  * @param[in] counter
 162  *   Pointer to the counter handler.
 163  */
 164 static void
 165 flow_verbs_counter_release(struct rte_eth_dev *dev,
 166                            struct mlx5_flow_counter *counter)
 167 {
 168         struct mlx5_priv *priv = dev->data->dev_private;
 169
 170         if (--counter->ref_cnt == 0) {
 171 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 172                 claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
 173 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 174                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 175 #endif
 176                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
 177                 rte_free(counter);
 178         }
 179 }
 180
 181 /**
 182  * Query a flow counter via Verbs library call.
 183  *
 184  * @see rte_flow_query()
 185  * @see rte_flow_ops
 186  */
 187 static int
 188 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 189                          struct rte_flow *flow, void *data,
 190                          struct rte_flow_error *error)
 191 {
 192 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 193         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 194         if (flow->counter && flow->counter->cs) {
 195                 struct rte_flow_query_count *qc = data;
 196                 uint64_t counters[2] = {0, 0};
 197 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 198                 struct ibv_query_counter_set_attr query_cs_attr = {
 199                         .cs = flow->counter->cs,
 200                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 201                 };
 202                 struct ibv_counter_set_data query_out = {
 203                         .out = counters,
 204                         .outlen = 2 * sizeof(uint64_t),
 205                 };
 206                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 207                                                        &query_out);
 208 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 209                 int err = mlx5_glue->query_counters
 210                                (flow->counter->cs, counters,
 211                                 RTE_DIM(counters),
 212                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 213 #endif
 214                 if (err)
 215                         return rte_flow_error_set
 216                                 (error, err,
 217                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 218                                  NULL,
 219                                  "cannot read counter");
 220                 qc->hits_set = 1;
 221                 qc->bytes_set = 1;
 222                 qc->hits = counters[0] - flow->counter->hits;
 223                 qc->bytes = counters[1] - flow->counter->bytes;
 224                 if (qc->reset) {
 225                         flow->counter->hits = counters[0];
 226                         flow->counter->bytes = counters[1];
 227                 }
 228                 return 0;
 229         }
 230         return rte_flow_error_set(error, EINVAL,
 231                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 232                                   NULL,
 233                                   "flow does not have counter");
 234 #else
 235         (void)flow;
 236         (void)data;
 237         return rte_flow_error_set(error, ENOTSUP,
 238                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 239                                   NULL,
 240                                   "counters are not available");
 241 #endif
 242 }
 243
 244 /**
 245  * Add a verbs item specification into @p verbs.
 246  *
 247  * @param[out] verbs
 248  *   Pointer to verbs structure.
 249  * @param[in] src
 250  *   Create specification.
 251  * @param[in] size
 252  *   Size in bytes of the specification to copy.
 253  */
 254 static void
 255 flow_verbs_spec_add(struct mlx5_flow_verbs *verbs, void *src, unsigned int size)
 256 {
 257         void *dst;
 258
 259         if (!verbs)
 260                 return;
 261         assert(verbs->specs);
 262         dst = (void *)(verbs->specs + verbs->size);
 263         memcpy(dst, src, size);
 264         ++verbs->attr->num_of_specs;
 265         verbs->size += size;
 266 }
 267
 268 /**
 269  * Convert the @p item into a Verbs specification. This function assumes that
 270  * the input is valid and that there is space to insert the requested item
 271  * into the flow.
 272  *
 273  * @param[in, out] dev_flow
 274  *   Pointer to dev_flow structure.
 275  * @param[in] item
 276  *   Item specification.
 277  * @param[in] item_flags
 278  *   Parsed item flags.
 279  */
 280 static void
 281 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 282                               const struct rte_flow_item *item,
 283                               uint64_t item_flags)
 284 {
 285         const struct rte_flow_item_eth *spec = item->spec;
 286         const struct rte_flow_item_eth *mask = item->mask;
 287         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 288         struct ibv_flow_spec_eth eth = {
 289                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 290                 .size = size,
 291         };
 292
 293         if (!mask)
 294                 mask = &rte_flow_item_eth_mask;
 295         if (spec) {
 296                 unsigned int i;
 297
 298                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 299                         RTE_ETHER_ADDR_LEN);
 300                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 301                         RTE_ETHER_ADDR_LEN);
 302                 eth.val.ether_type = spec->type;
 303                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 304                         RTE_ETHER_ADDR_LEN);
 305                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 306                         RTE_ETHER_ADDR_LEN);
 307                 eth.mask.ether_type = mask->type;
 308                 /* Remove unwanted bits from values. */
 309                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 310                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 311                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 312                 }
 313                 eth.val.ether_type &= eth.mask.ether_type;
 314         }
 315         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 316 }
 317
 318 /**
 319  * Update the VLAN tag in the Verbs Ethernet specification.
 320  * This function assumes that the input is valid and there is space to add
 321  * the requested item.
 322  *
 323  * @param[in, out] attr
 324  *   Pointer to Verbs attributes structure.
 325  * @param[in] eth
 326  *   Verbs structure containing the VLAN information to copy.
 327  */
 328 static void
 329 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 330                             struct ibv_flow_spec_eth *eth)
 331 {
 332         unsigned int i;
 333         const enum ibv_flow_spec_type search = eth->type;
 334         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 335                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 336
 337         for (i = 0; i != attr->num_of_specs; ++i) {
 338                 if (hdr->type == search) {
 339                         struct ibv_flow_spec_eth *e =
 340                                 (struct ibv_flow_spec_eth *)hdr;
 341
 342                         e->val.vlan_tag = eth->val.vlan_tag;
 343                         e->mask.vlan_tag = eth->mask.vlan_tag;
 344                         e->val.ether_type = eth->val.ether_type;
 345                         e->mask.ether_type = eth->mask.ether_type;
 346                         break;
 347                 }
 348                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 349         }
 350 }
 351
 352 /**
 353  * Convert the @p item into a Verbs specification. This function assumes that
 354  * the input is valid and that there is space to insert the requested item
 355  * into the flow.
 356  *
 357  * @param[in, out] dev_flow
 358  *   Pointer to dev_flow structure.
 359  * @param[in] item
 360  *   Item specification.
 361  * @param[in] item_flags
 362  *   Parsed item flags.
 363  */
 364 static void
 365 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 366                                const struct rte_flow_item *item,
 367                                uint64_t item_flags)
 368 {
 369         const struct rte_flow_item_vlan *spec = item->spec;
 370         const struct rte_flow_item_vlan *mask = item->mask;
 371         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 372         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 373         struct ibv_flow_spec_eth eth = {
 374                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 375                 .size = size,
 376         };
 377         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 378                                       MLX5_FLOW_LAYER_OUTER_L2;
 379
 380         if (!mask)
 381                 mask = &rte_flow_item_vlan_mask;
 382         if (spec) {
 383                 eth.val.vlan_tag = spec->tci;
 384                 eth.mask.vlan_tag = mask->tci;
 385                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 386                 eth.val.ether_type = spec->inner_type;
 387                 eth.mask.ether_type = mask->inner_type;
 388                 eth.val.ether_type &= eth.mask.ether_type;
 389         }
 390         if (!(item_flags & l2m))
 391                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 392         else
 393                 flow_verbs_item_vlan_update(dev_flow->verbs.attr, &eth);
 394         if (!tunnel)
 395                 dev_flow->verbs.vf_vlan.tag =
 396                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 397 }
 398
 399 /**
 400  * Convert the @p item into a Verbs specification. This function assumes that
 401  * the input is valid and that there is space to insert the requested item
 402  * into the flow.
 403  *
 404  * @param[in, out] dev_flow
 405  *   Pointer to dev_flow structure.
 406  * @param[in] item
 407  *   Item specification.
 408  * @param[in] item_flags
 409  *   Parsed item flags.
 410  */
 411 static void
 412 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 413                                const struct rte_flow_item *item,
 414                                uint64_t item_flags)
 415 {
 416         const struct rte_flow_item_ipv4 *spec = item->spec;
 417         const struct rte_flow_item_ipv4 *mask = item->mask;
 418         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 419         struct ibv_flow_spec_ipv4_ext ipv4 = {
 420                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 421                 .size = size,
 422         };
 423
 424         if (!mask)
 425                 mask = &rte_flow_item_ipv4_mask;
 426         if (spec) {
 427                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 428                         .src_ip = spec->hdr.src_addr,
 429                         .dst_ip = spec->hdr.dst_addr,
 430                         .proto = spec->hdr.next_proto_id,
 431                         .tos = spec->hdr.type_of_service,
 432                 };
 433                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 434                         .src_ip = mask->hdr.src_addr,
 435                         .dst_ip = mask->hdr.dst_addr,
 436                         .proto = mask->hdr.next_proto_id,
 437                         .tos = mask->hdr.type_of_service,
 438                 };
 439                 /* Remove unwanted bits from values. */
 440                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 441                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 442                 ipv4.val.proto &= ipv4.mask.proto;
 443                 ipv4.val.tos &= ipv4.mask.tos;
 444         }
 445         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 446 }
 447
 448 /**
 449  * Convert the @p item into a Verbs specification. This function assumes that
 450  * the input is valid and that there is space to insert the requested item
 451  * into the flow.
 452  *
 453  * @param[in, out] dev_flow
 454  *   Pointer to dev_flow structure.
 455  * @param[in] item
 456  *   Item specification.
 457  * @param[in] item_flags
 458  *   Parsed item flags.
 459  */
 460 static void
 461 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 462                                const struct rte_flow_item *item,
 463                                uint64_t item_flags)
 464 {
 465         const struct rte_flow_item_ipv6 *spec = item->spec;
 466         const struct rte_flow_item_ipv6 *mask = item->mask;
 467         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 468         struct ibv_flow_spec_ipv6 ipv6 = {
 469                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 470                 .size = size,
 471         };
 472
 473         if (!mask)
 474                 mask = &rte_flow_item_ipv6_mask;
 475         if (spec) {
 476                 unsigned int i;
 477                 uint32_t vtc_flow_val;
 478                 uint32_t vtc_flow_mask;
 479
 480                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 481                        RTE_DIM(ipv6.val.src_ip));
 482                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 483                        RTE_DIM(ipv6.val.dst_ip));
 484                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 485                        RTE_DIM(ipv6.mask.src_ip));
 486                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 487                        RTE_DIM(ipv6.mask.dst_ip));
 488                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 489                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 490                 ipv6.val.flow_label =
 491                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 492                                          RTE_IPV6_HDR_FL_SHIFT);
 493                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 494                                          RTE_IPV6_HDR_TC_SHIFT;
 495                 ipv6.val.next_hdr = spec->hdr.proto;
 496                 ipv6.val.hop_limit = spec->hdr.hop_limits;
 497                 ipv6.mask.flow_label =
 498                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 499                                          RTE_IPV6_HDR_FL_SHIFT);
 500                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 501                                           RTE_IPV6_HDR_TC_SHIFT;
 502                 ipv6.mask.next_hdr = mask->hdr.proto;
 503                 ipv6.mask.hop_limit = mask->hdr.hop_limits;
 504                 /* Remove unwanted bits from values. */
 505                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 506                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 507                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 508                 }
 509                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 510                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 511                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 512                 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 513         }
 514         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 515 }
 516
 517 /**
 518  * Convert the @p item into a Verbs specification. This function assumes that
 519  * the input is valid and that there is space to insert the requested item
 520  * into the flow.
 521  *
 522  * @param[in, out] dev_flow
 523  *   Pointer to dev_flow structure.
 524  * @param[in] item
 525  *   Item specification.
 526  * @param[in] item_flags
 527  *   Parsed item flags.
 528  */
 529 static void
 530 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 531                               const struct rte_flow_item *item,
 532                               uint64_t item_flags __rte_unused)
 533 {
 534         const struct rte_flow_item_tcp *spec = item->spec;
 535         const struct rte_flow_item_tcp *mask = item->mask;
 536         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 537         struct ibv_flow_spec_tcp_udp tcp = {
 538                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 539                 .size = size,
 540         };
 541
 542         if (!mask)
 543                 mask = &rte_flow_item_tcp_mask;
 544         if (spec) {
 545                 tcp.val.dst_port = spec->hdr.dst_port;
 546                 tcp.val.src_port = spec->hdr.src_port;
 547                 tcp.mask.dst_port = mask->hdr.dst_port;
 548                 tcp.mask.src_port = mask->hdr.src_port;
 549                 /* Remove unwanted bits from values. */
 550                 tcp.val.src_port &= tcp.mask.src_port;
 551                 tcp.val.dst_port &= tcp.mask.dst_port;
 552         }
 553         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 554 }
 555
 556 /**
 557  * Convert the @p item into a Verbs specification. This function assumes that
 558  * the input is valid and that there is space to insert the requested item
 559  * into the flow.
 560  *
 561  * @param[in, out] dev_flow
 562  *   Pointer to dev_flow structure.
 563  * @param[in] item
 564  *   Item specification.
 565  * @param[in] item_flags
 566  *   Parsed item flags.
 567  */
 568 static void
 569 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 570                               const struct rte_flow_item *item,
 571                               uint64_t item_flags __rte_unused)
 572 {
 573         const struct rte_flow_item_udp *spec = item->spec;
 574         const struct rte_flow_item_udp *mask = item->mask;
 575         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 576         struct ibv_flow_spec_tcp_udp udp = {
 577                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 578                 .size = size,
 579         };
 580
 581         if (!mask)
 582                 mask = &rte_flow_item_udp_mask;
 583         if (spec) {
 584                 udp.val.dst_port = spec->hdr.dst_port;
 585                 udp.val.src_port = spec->hdr.src_port;
 586                 udp.mask.dst_port = mask->hdr.dst_port;
 587                 udp.mask.src_port = mask->hdr.src_port;
 588                 /* Remove unwanted bits from values. */
 589                 udp.val.src_port &= udp.mask.src_port;
 590                 udp.val.dst_port &= udp.mask.dst_port;
 591         }
 592         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 593 }
 594
 595 /**
 596  * Convert the @p item into a Verbs specification. This function assumes that
 597  * the input is valid and that there is space to insert the requested item
 598  * into the flow.
 599  *
 600  * @param[in, out] dev_flow
 601  *   Pointer to dev_flow structure.
 602  * @param[in] item
 603  *   Item specification.
 604  * @param[in] item_flags
 605  *   Parsed item flags.
 606  */
 607 static void
 608 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 609                                 const struct rte_flow_item *item,
 610                                 uint64_t item_flags __rte_unused)
 611 {
 612         const struct rte_flow_item_vxlan *spec = item->spec;
 613         const struct rte_flow_item_vxlan *mask = item->mask;
 614         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 615         struct ibv_flow_spec_tunnel vxlan = {
 616                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 617                 .size = size,
 618         };
 619         union vni {
 620                 uint32_t vlan_id;
 621                 uint8_t vni[4];
 622         } id = { .vlan_id = 0, };
 623
 624         if (!mask)
 625                 mask = &rte_flow_item_vxlan_mask;
 626         if (spec) {
 627                 memcpy(&id.vni[1], spec->vni, 3);
 628                 vxlan.val.tunnel_id = id.vlan_id;
 629                 memcpy(&id.vni[1], mask->vni, 3);
 630                 vxlan.mask.tunnel_id = id.vlan_id;
 631                 /* Remove unwanted bits from values. */
 632                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 633         }
 634         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 635 }
 636
 637 /**
 638  * Convert the @p item into a Verbs specification. This function assumes that
 639  * the input is valid and that there is space to insert the requested item
 640  * into the flow.
 641  *
 642  * @param[in, out] dev_flow
 643  *   Pointer to dev_flow structure.
 644  * @param[in] item
 645  *   Item specification.
 646  * @param[in] item_flags
 647  *   Parsed item flags.
 648  */
 649 static void
 650 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 651                                     const struct rte_flow_item *item,
 652                                     uint64_t item_flags __rte_unused)
 653 {
 654         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 655         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 656         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 657         struct ibv_flow_spec_tunnel vxlan_gpe = {
 658                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 659                 .size = size,
 660         };
 661         union vni {
 662                 uint32_t vlan_id;
 663                 uint8_t vni[4];
 664         } id = { .vlan_id = 0, };
 665
 666         if (!mask)
 667                 mask = &rte_flow_item_vxlan_gpe_mask;
 668         if (spec) {
 669                 memcpy(&id.vni[1], spec->vni, 3);
 670                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 671                 memcpy(&id.vni[1], mask->vni, 3);
 672                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 673                 /* Remove unwanted bits from values. */
 674                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 675         }
 676         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 677 }
 678
 679 /**
 680  * Update the protocol in Verbs IPv4/IPv6 spec.
 681  *
 682  * @param[in, out] attr
 683  *   Pointer to Verbs attributes structure.
 684  * @param[in] search
 685  *   Specification type to search in order to update the IP protocol.
 686  * @param[in] protocol
 687  *   Protocol value to set if none is present in the specification.
 688  */
 689 static void
 690 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 691                                        enum ibv_flow_spec_type search,
 692                                        uint8_t protocol)
 693 {
 694         unsigned int i;
 695         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 696                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 697
 698         if (!attr)
 699                 return;
 700         for (i = 0; i != attr->num_of_specs; ++i) {
 701                 if (hdr->type == search) {
 702                         union {
 703                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 704                                 struct ibv_flow_spec_ipv6 *ipv6;
 705                         } ip;
 706
 707                         switch (search) {
 708                         case IBV_FLOW_SPEC_IPV4_EXT:
 709                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 710                                 if (!ip.ipv4->val.proto) {
 711                                         ip.ipv4->val.proto = protocol;
 712                                         ip.ipv4->mask.proto = 0xff;
 713                                 }
 714                                 break;
 715                         case IBV_FLOW_SPEC_IPV6:
 716                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 717                                 if (!ip.ipv6->val.next_hdr) {
 718                                         ip.ipv6->val.next_hdr = protocol;
 719                                         ip.ipv6->mask.next_hdr = 0xff;
 720                                 }
 721                                 break;
 722                         default:
 723                                 break;
 724                         }
 725                         break;
 726                 }
 727                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 728         }
 729 }
 730
 731 /**
 732  * Convert the @p item into a Verbs specification. This function assumes that
 733  * the input is valid and that there is space to insert the requested item
 734  * into the flow.
 735  *
 736  * @param[in, out] dev_flow
 737  *   Pointer to dev_flow structure.
 738  * @param[in] item
 739  *   Item specification.
 740  * @param[in] item_flags
 741  *   Parsed item flags.
 742  */
 743 static void
 744 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 745                               const struct rte_flow_item *item __rte_unused,
 746                               uint64_t item_flags)
 747 {
 748         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
 749 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 750         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 751         struct ibv_flow_spec_tunnel tunnel = {
 752                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 753                 .size = size,
 754         };
 755 #else
 756         const struct rte_flow_item_gre *spec = item->spec;
 757         const struct rte_flow_item_gre *mask = item->mask;
 758         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 759         struct ibv_flow_spec_gre tunnel = {
 760                 .type = IBV_FLOW_SPEC_GRE,
 761                 .size = size,
 762         };
 763
 764         if (!mask)
 765                 mask = &rte_flow_item_gre_mask;
 766         if (spec) {
 767                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 768                 tunnel.val.protocol = spec->protocol;
 769                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 770                 tunnel.mask.protocol = mask->protocol;
 771                 /* Remove unwanted bits from values. */
 772                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 773                 tunnel.val.protocol &= tunnel.mask.protocol;
 774                 tunnel.val.key &= tunnel.mask.key;
 775         }
 776 #endif
 777         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 778                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 779                                                        IBV_FLOW_SPEC_IPV4_EXT,
 780                                                        IPPROTO_GRE);
 781         else
 782                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 783                                                        IBV_FLOW_SPEC_IPV6,
 784                                                        IPPROTO_GRE);
 785         flow_verbs_spec_add(verbs, &tunnel, size);
 786 }
 787
 788 /**
 789  * Convert the @p action into a Verbs specification. This function assumes that
 790  * the input is valid and that there is space to insert the requested action
 791  * into the flow. This function also return the action that was added.
 792  *
 793  * @param[in, out] dev_flow
 794  *   Pointer to dev_flow structure.
 795  * @param[in] item
 796  *   Item specification.
 797  * @param[in] item_flags
 798  *   Parsed item flags.
 799  */
 800 static void
 801 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
 802                                const struct rte_flow_item *item __rte_unused,
 803                                uint64_t item_flags __rte_unused)
 804 {
 805 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 806         const struct rte_flow_item_mpls *spec = item->spec;
 807         const struct rte_flow_item_mpls *mask = item->mask;
 808         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 809         struct ibv_flow_spec_mpls mpls = {
 810                 .type = IBV_FLOW_SPEC_MPLS,
 811                 .size = size,
 812         };
 813
 814         if (!mask)
 815                 mask = &rte_flow_item_mpls_mask;
 816         if (spec) {
 817                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 818                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 819                 /* Remove unwanted bits from values.  */
 820                 mpls.val.label &= mpls.mask.label;
 821         }
 822         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
 823 #endif
 824 }
 825
 826 /**
 827  * Convert the @p action into a Verbs specification. This function assumes that
 828  * the input is valid and that there is space to insert the requested action
 829  * into the flow.
 830  *
 831  * @param[in] dev_flow
 832  *   Pointer to mlx5_flow.
 833  * @param[in] action
 834  *   Action configuration.
 835  */
 836 static void
 837 flow_verbs_translate_action_drop
 838         (struct mlx5_flow *dev_flow,
 839          const struct rte_flow_action *action __rte_unused)
 840 {
 841         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 842         struct ibv_flow_spec_action_drop drop = {
 843                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 844                         .size = size,
 845         };
 846
 847         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
 848 }
 849
 850 /**
 851  * Convert the @p action into a Verbs specification. This function assumes that
 852  * the input is valid and that there is space to insert the requested action
 853  * into the flow.
 854  *
 855  * @param[in] dev_flow
 856  *   Pointer to mlx5_flow.
 857  * @param[in] action
 858  *   Action configuration.
 859  */
 860 static void
 861 flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow,
 862                                   const struct rte_flow_action *action)
 863 {
 864         const struct rte_flow_action_queue *queue = action->conf;
 865         struct rte_flow *flow = dev_flow->flow;
 866
 867         if (flow->rss.queue)
 868                 (*flow->rss.queue)[0] = queue->index;
 869         flow->rss.queue_num = 1;
 870 }
 871
 872 /**
 873  * Convert the @p action into a Verbs specification. This function assumes that
 874  * the input is valid and that there is space to insert the requested action
 875  * into the flow.
 876  *
 877  * @param[in] action
 878  *   Action configuration.
 879  * @param[in, out] action_flags
 880  *   Pointer to the detected actions.
 881  * @param[in] dev_flow
 882  *   Pointer to mlx5_flow.
 883  */
 884 static void
 885 flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow,
 886                                 const struct rte_flow_action *action)
 887 {
 888         const struct rte_flow_action_rss *rss = action->conf;
 889         const uint8_t *rss_key;
 890         struct rte_flow *flow = dev_flow->flow;
 891
 892         if (flow->rss.queue)
 893                 memcpy((*flow->rss.queue), rss->queue,
 894                        rss->queue_num * sizeof(uint16_t));
 895         flow->rss.queue_num = rss->queue_num;
 896         /* NULL RSS key indicates default RSS key. */
 897         rss_key = !rss->key ? rss_hash_default_key : rss->key;
 898         memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN);
 899         /*
 900          * rss->level and rss.types should be set in advance when expanding
 901          * items for RSS.
 902          */
 903 }
 904
 905 /**
 906  * Convert the @p action into a Verbs specification. This function assumes that
 907  * the input is valid and that there is space to insert the requested action
 908  * into the flow.
 909  *
 910  * @param[in] dev_flow
 911  *   Pointer to mlx5_flow.
 912  * @param[in] action
 913  *   Action configuration.
 914  */
 915 static void
 916 flow_verbs_translate_action_flag
 917         (struct mlx5_flow *dev_flow,
 918          const struct rte_flow_action *action __rte_unused)
 919 {
 920         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 921         struct ibv_flow_spec_action_tag tag = {
 922                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 923                 .size = size,
 924                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 925         };
 926
 927         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 928 }
 929
 930 /**
 931  * Convert the @p action into a Verbs specification. This function assumes that
 932  * the input is valid and that there is space to insert the requested action
 933  * into the flow.
 934  *
 935  * @param[in] dev_flow
 936  *   Pointer to mlx5_flow.
 937  * @param[in] action
 938  *   Action configuration.
 939  */
 940 static void
 941 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
 942                                  const struct rte_flow_action *action)
 943 {
 944         const struct rte_flow_action_mark *mark = action->conf;
 945         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 946         struct ibv_flow_spec_action_tag tag = {
 947                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 948                 .size = size,
 949                 .tag_id = mlx5_flow_mark_set(mark->id),
 950         };
 951
 952         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 953 }
 954
 955 /**
 956  * Convert the @p action into a Verbs specification. This function assumes that
 957  * the input is valid and that there is space to insert the requested action
 958  * into the flow.
 959  *
 960  * @param[in] dev
 961  *   Pointer to the Ethernet device structure.
 962  * @param[in] action
 963  *   Action configuration.
 964  * @param[in] dev_flow
 965  *   Pointer to mlx5_flow.
 966  * @param[out] error
 967  *   Pointer to error structure.
 968  *
 969  * @return
 970  *   0 On success else a negative errno value is returned and rte_errno is set.
 971  */
 972 static int
 973 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
 974                                   const struct rte_flow_action *action,
 975                                   struct rte_eth_dev *dev,
 976                                   struct rte_flow_error *error)
 977 {
 978         const struct rte_flow_action_count *count = action->conf;
 979         struct rte_flow *flow = dev_flow->flow;
 980 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 981         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 982         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 983         struct ibv_flow_spec_counter_action counter = {
 984                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
 985                 .size = size,
 986         };
 987 #endif
 988
 989         if (!flow->counter) {
 990                 flow->counter = flow_verbs_counter_new(dev, count->shared,
 991                                                        count->id);
 992                 if (!flow->counter)
 993                         return rte_flow_error_set(error, rte_errno,
 994                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 995                                                   action,
 996                                                   "cannot get counter"
 997                                                   " context.");
 998         }
 999 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1000         counter.counter_set_handle = flow->counter->cs->handle;
1001         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1002 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1003         counter.counters = flow->counter->cs;
1004         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1005 #endif
1006         return 0;
1007 }
1008
1009 /**
1010  * Internal validation function. For validating both actions and items.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the Ethernet device structure.
1014  * @param[in] attr
1015  *   Pointer to the flow attributes.
1016  * @param[in] items
1017  *   Pointer to the list of items.
1018  * @param[in] actions
1019  *   Pointer to the list of actions.
1020  * @param[in] external
1021  *   This flow rule is created by request external to PMD.
1022  * @param[out] error
1023  *   Pointer to the error structure.
1024  *
1025  * @return
1026  *   0 on success, a negative errno value otherwise and rte_errno is set.
1027  */
1028 static int
1029 flow_verbs_validate(struct rte_eth_dev *dev,
1030                     const struct rte_flow_attr *attr,
1031                     const struct rte_flow_item items[],
1032                     const struct rte_flow_action actions[],
1033                     bool external __rte_unused,
1034                     struct rte_flow_error *error)
1035 {
1036         int ret;
1037         uint64_t action_flags = 0;
1038         uint64_t item_flags = 0;
1039         uint64_t last_item = 0;
1040         uint8_t next_protocol = 0xff;
1041         uint16_t ether_type = 0;
1042
1043         if (items == NULL)
1044                 return -1;
1045         ret = mlx5_flow_validate_attributes(dev, attr, error);
1046         if (ret < 0)
1047                 return ret;
1048         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1049                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1050                 int ret = 0;
1051
1052                 switch (items->type) {
1053                 case RTE_FLOW_ITEM_TYPE_VOID:
1054                         break;
1055                 case RTE_FLOW_ITEM_TYPE_ETH:
1056                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1057                                                           error);
1058                         if (ret < 0)
1059                                 return ret;
1060                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1061                                              MLX5_FLOW_LAYER_OUTER_L2;
1062                         if (items->mask != NULL && items->spec != NULL) {
1063                                 ether_type =
1064                                         ((const struct rte_flow_item_eth *)
1065                                          items->spec)->type;
1066                                 ether_type &=
1067                                         ((const struct rte_flow_item_eth *)
1068                                          items->mask)->type;
1069                                 ether_type = rte_be_to_cpu_16(ether_type);
1070                         } else {
1071                                 ether_type = 0;
1072                         }
1073                         break;
1074                 case RTE_FLOW_ITEM_TYPE_VLAN:
1075                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1076                                                            dev, error);
1077                         if (ret < 0)
1078                                 return ret;
1079                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1080                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1081                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1082                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1083                         if (items->mask != NULL && items->spec != NULL) {
1084                                 ether_type =
1085                                         ((const struct rte_flow_item_vlan *)
1086                                          items->spec)->inner_type;
1087                                 ether_type &=
1088                                         ((const struct rte_flow_item_vlan *)
1089                                          items->mask)->inner_type;
1090                                 ether_type = rte_be_to_cpu_16(ether_type);
1091                         } else {
1092                                 ether_type = 0;
1093                         }
1094                         break;
1095                 case RTE_FLOW_ITEM_TYPE_IPV4:
1096                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1097                                                            last_item,
1098                                                            ether_type, NULL,
1099                                                            error);
1100                         if (ret < 0)
1101                                 return ret;
1102                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1103                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1104                         if (items->mask != NULL &&
1105                             ((const struct rte_flow_item_ipv4 *)
1106                              items->mask)->hdr.next_proto_id) {
1107                                 next_protocol =
1108                                         ((const struct rte_flow_item_ipv4 *)
1109                                          (items->spec))->hdr.next_proto_id;
1110                                 next_protocol &=
1111                                         ((const struct rte_flow_item_ipv4 *)
1112                                          (items->mask))->hdr.next_proto_id;
1113                         } else {
1114                                 /* Reset for inner layer. */
1115                                 next_protocol = 0xff;
1116                         }
1117                         break;
1118                 case RTE_FLOW_ITEM_TYPE_IPV6:
1119                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1120                                                            last_item,
1121                                                            ether_type, NULL,
1122                                                            error);
1123                         if (ret < 0)
1124                                 return ret;
1125                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1126                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1127                         if (items->mask != NULL &&
1128                             ((const struct rte_flow_item_ipv6 *)
1129                              items->mask)->hdr.proto) {
1130                                 next_protocol =
1131                                         ((const struct rte_flow_item_ipv6 *)
1132                                          items->spec)->hdr.proto;
1133                                 next_protocol &=
1134                                         ((const struct rte_flow_item_ipv6 *)
1135                                          items->mask)->hdr.proto;
1136                         } else {
1137                                 /* Reset for inner layer. */
1138                                 next_protocol = 0xff;
1139                         }
1140                         break;
1141                 case RTE_FLOW_ITEM_TYPE_UDP:
1142                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1143                                                           next_protocol,
1144                                                           error);
1145                         if (ret < 0)
1146                                 return ret;
1147                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1148                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1149                         break;
1150                 case RTE_FLOW_ITEM_TYPE_TCP:
1151                         ret = mlx5_flow_validate_item_tcp
1152                                                 (items, item_flags,
1153                                                  next_protocol,
1154                                                  &rte_flow_item_tcp_mask,
1155                                                  error);
1156                         if (ret < 0)
1157                                 return ret;
1158                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1159                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1160                         break;
1161                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1162                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1163                                                             error);
1164                         if (ret < 0)
1165                                 return ret;
1166                         last_item = MLX5_FLOW_LAYER_VXLAN;
1167                         break;
1168                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1169                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1170                                                                 item_flags,
1171                                                                 dev, error);
1172                         if (ret < 0)
1173                                 return ret;
1174                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1175                         break;
1176                 case RTE_FLOW_ITEM_TYPE_GRE:
1177                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1178                                                           next_protocol, error);
1179                         if (ret < 0)
1180                                 return ret;
1181                         last_item = MLX5_FLOW_LAYER_GRE;
1182                         break;
1183                 case RTE_FLOW_ITEM_TYPE_MPLS:
1184                         ret = mlx5_flow_validate_item_mpls(dev, items,
1185                                                            item_flags,
1186                                                            last_item, error);
1187                         if (ret < 0)
1188                                 return ret;
1189                         last_item = MLX5_FLOW_LAYER_MPLS;
1190                         break;
1191                 default:
1192                         return rte_flow_error_set(error, ENOTSUP,
1193                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1194                                                   NULL, "item not supported");
1195                 }
1196                 item_flags |= last_item;
1197         }
1198         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1199                 switch (actions->type) {
1200                 case RTE_FLOW_ACTION_TYPE_VOID:
1201                         break;
1202                 case RTE_FLOW_ACTION_TYPE_FLAG:
1203                         ret = mlx5_flow_validate_action_flag(action_flags,
1204                                                              attr,
1205                                                              error);
1206                         if (ret < 0)
1207                                 return ret;
1208                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1209                         break;
1210                 case RTE_FLOW_ACTION_TYPE_MARK:
1211                         ret = mlx5_flow_validate_action_mark(actions,
1212                                                              action_flags,
1213                                                              attr,
1214                                                              error);
1215                         if (ret < 0)
1216                                 return ret;
1217                         action_flags |= MLX5_FLOW_ACTION_MARK;
1218                         break;
1219                 case RTE_FLOW_ACTION_TYPE_DROP:
1220                         ret = mlx5_flow_validate_action_drop(action_flags,
1221                                                              attr,
1222                                                              error);
1223                         if (ret < 0)
1224                                 return ret;
1225                         action_flags |= MLX5_FLOW_ACTION_DROP;
1226                         break;
1227                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1228                         ret = mlx5_flow_validate_action_queue(actions,
1229                                                               action_flags, dev,
1230                                                               attr,
1231                                                               error);
1232                         if (ret < 0)
1233                                 return ret;
1234                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1235                         break;
1236                 case RTE_FLOW_ACTION_TYPE_RSS:
1237                         ret = mlx5_flow_validate_action_rss(actions,
1238                                                             action_flags, dev,
1239                                                             attr, item_flags,
1240                                                             error);
1241                         if (ret < 0)
1242                                 return ret;
1243                         action_flags |= MLX5_FLOW_ACTION_RSS;
1244                         break;
1245                 case RTE_FLOW_ACTION_TYPE_COUNT:
1246                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1247                         if (ret < 0)
1248                                 return ret;
1249                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1250                         break;
1251                 default:
1252                         return rte_flow_error_set(error, ENOTSUP,
1253                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1254                                                   actions,
1255                                                   "action not supported");
1256                 }
1257         }
1258         /*
1259          * Validate the drop action mutual exclusion with other actions.
1260          * Drop action is mutually-exclusive with any other action, except for
1261          * Count action.
1262          */
1263         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1264             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1265                 return rte_flow_error_set(error, EINVAL,
1266                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1267                                           "Drop action is mutually-exclusive "
1268                                           "with any other action, except for "
1269                                           "Count action");
1270         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1271                 return rte_flow_error_set(error, EINVAL,
1272                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1273                                           "no fate action is found");
1274         return 0;
1275 }
1276
1277 /**
1278  * Calculate the required bytes that are needed for the action part of the verbs
1279  * flow.
1280  *
1281  * @param[in] actions
1282  *   Pointer to the list of actions.
1283  *
1284  * @return
1285  *   The size of the memory needed for all actions.
1286  */
1287 static int
1288 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1289 {
1290         int size = 0;
1291
1292         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1293                 switch (actions->type) {
1294                 case RTE_FLOW_ACTION_TYPE_VOID:
1295                         break;
1296                 case RTE_FLOW_ACTION_TYPE_FLAG:
1297                         size += sizeof(struct ibv_flow_spec_action_tag);
1298                         break;
1299                 case RTE_FLOW_ACTION_TYPE_MARK:
1300                         size += sizeof(struct ibv_flow_spec_action_tag);
1301                         break;
1302                 case RTE_FLOW_ACTION_TYPE_DROP:
1303                         size += sizeof(struct ibv_flow_spec_action_drop);
1304                         break;
1305                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1306                         break;
1307                 case RTE_FLOW_ACTION_TYPE_RSS:
1308                         break;
1309                 case RTE_FLOW_ACTION_TYPE_COUNT:
1310 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1311         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1312                         size += sizeof(struct ibv_flow_spec_counter_action);
1313 #endif
1314                         break;
1315                 default:
1316                         break;
1317                 }
1318         }
1319         return size;
1320 }
1321
1322 /**
1323  * Calculate the required bytes that are needed for the item part of the verbs
1324  * flow.
1325  *
1326  * @param[in] items
1327  *   Pointer to the list of items.
1328  *
1329  * @return
1330  *   The size of the memory needed for all items.
1331  */
1332 static int
1333 flow_verbs_get_items_size(const struct rte_flow_item items[])
1334 {
1335         int size = 0;
1336
1337         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1338                 switch (items->type) {
1339                 case RTE_FLOW_ITEM_TYPE_VOID:
1340                         break;
1341                 case RTE_FLOW_ITEM_TYPE_ETH:
1342                         size += sizeof(struct ibv_flow_spec_eth);
1343                         break;
1344                 case RTE_FLOW_ITEM_TYPE_VLAN:
1345                         size += sizeof(struct ibv_flow_spec_eth);
1346                         break;
1347                 case RTE_FLOW_ITEM_TYPE_IPV4:
1348                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1349                         break;
1350                 case RTE_FLOW_ITEM_TYPE_IPV6:
1351                         size += sizeof(struct ibv_flow_spec_ipv6);
1352                         break;
1353                 case RTE_FLOW_ITEM_TYPE_UDP:
1354                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1355                         break;
1356                 case RTE_FLOW_ITEM_TYPE_TCP:
1357                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1358                         break;
1359                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1360                         size += sizeof(struct ibv_flow_spec_tunnel);
1361                         break;
1362                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1363                         size += sizeof(struct ibv_flow_spec_tunnel);
1364                         break;
1365 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1366                 case RTE_FLOW_ITEM_TYPE_GRE:
1367                         size += sizeof(struct ibv_flow_spec_gre);
1368                         break;
1369                 case RTE_FLOW_ITEM_TYPE_MPLS:
1370                         size += sizeof(struct ibv_flow_spec_mpls);
1371                         break;
1372 #else
1373                 case RTE_FLOW_ITEM_TYPE_GRE:
1374                         size += sizeof(struct ibv_flow_spec_tunnel);
1375                         break;
1376 #endif
1377                 default:
1378                         break;
1379                 }
1380         }
1381         return size;
1382 }
1383
1384 /**
1385  * Internal preparation function. Allocate mlx5_flow with the required size.
1386  * The required size is calculate based on the actions and items. This function
1387  * also returns the detected actions and items for later use.
1388  *
1389  * @param[in] attr
1390  *   Pointer to the flow attributes.
1391  * @param[in] items
1392  *   Pointer to the list of items.
1393  * @param[in] actions
1394  *   Pointer to the list of actions.
1395  * @param[out] error
1396  *   Pointer to the error structure.
1397  *
1398  * @return
1399  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1400  *   is set.
1401  */
1402 static struct mlx5_flow *
1403 flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
1404                    const struct rte_flow_item items[],
1405                    const struct rte_flow_action actions[],
1406                    struct rte_flow_error *error)
1407 {
1408         size_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
1409         struct mlx5_flow *dev_flow;
1410
1411         size += flow_verbs_get_actions_size(actions);
1412         size += flow_verbs_get_items_size(items);
1413         dev_flow = rte_calloc(__func__, 1, size, 0);
1414         if (!dev_flow) {
1415                 rte_flow_error_set(error, ENOMEM,
1416                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1417                                    "not enough memory to create flow");
1418                 return NULL;
1419         }
1420         dev_flow->verbs.attr = (void *)(dev_flow + 1);
1421         dev_flow->verbs.specs = (void *)(dev_flow->verbs.attr + 1);
1422         dev_flow->ingress = attr->ingress;
1423         dev_flow->transfer = attr->transfer;
1424         return dev_flow;
1425 }
1426
1427 /**
1428  * Fill the flow with verb spec.
1429  *
1430  * @param[in] dev
1431  *   Pointer to Ethernet device.
1432  * @param[in, out] dev_flow
1433  *   Pointer to the mlx5 flow.
1434  * @param[in] attr
1435  *   Pointer to the flow attributes.
1436  * @param[in] items
1437  *   Pointer to the list of items.
1438  * @param[in] actions
1439  *   Pointer to the list of actions.
1440  * @param[out] error
1441  *   Pointer to the error structure.
1442  *
1443  * @return
1444  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1445  */
1446 static int
1447 flow_verbs_translate(struct rte_eth_dev *dev,
1448                      struct mlx5_flow *dev_flow,
1449                      const struct rte_flow_attr *attr,
1450                      const struct rte_flow_item items[],
1451                      const struct rte_flow_action actions[],
1452                      struct rte_flow_error *error)
1453 {
1454         uint64_t item_flags = 0;
1455         uint64_t action_flags = 0;
1456         uint64_t priority = attr->priority;
1457         uint32_t subpriority = 0;
1458         struct mlx5_priv *priv = dev->data->dev_private;
1459
1460         if (priority == MLX5_FLOW_PRIO_RSVD)
1461                 priority = priv->config.flow_prio - 1;
1462         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1463                 int ret;
1464
1465                 switch (actions->type) {
1466                 case RTE_FLOW_ACTION_TYPE_VOID:
1467                         break;
1468                 case RTE_FLOW_ACTION_TYPE_FLAG:
1469                         flow_verbs_translate_action_flag(dev_flow, actions);
1470                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1471                         break;
1472                 case RTE_FLOW_ACTION_TYPE_MARK:
1473                         flow_verbs_translate_action_mark(dev_flow, actions);
1474                         action_flags |= MLX5_FLOW_ACTION_MARK;
1475                         break;
1476                 case RTE_FLOW_ACTION_TYPE_DROP:
1477                         flow_verbs_translate_action_drop(dev_flow, actions);
1478                         action_flags |= MLX5_FLOW_ACTION_DROP;
1479                         break;
1480                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1481                         flow_verbs_translate_action_queue(dev_flow, actions);
1482                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1483                         break;
1484                 case RTE_FLOW_ACTION_TYPE_RSS:
1485                         flow_verbs_translate_action_rss(dev_flow, actions);
1486                         action_flags |= MLX5_FLOW_ACTION_RSS;
1487                         break;
1488                 case RTE_FLOW_ACTION_TYPE_COUNT:
1489                         ret = flow_verbs_translate_action_count(dev_flow,
1490                                                                 actions,
1491                                                                 dev, error);
1492                         if (ret < 0)
1493                                 return ret;
1494                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1495                         break;
1496                 default:
1497                         return rte_flow_error_set(error, ENOTSUP,
1498                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1499                                                   actions,
1500                                                   "action not supported");
1501                 }
1502         }
1503         dev_flow->actions = action_flags;
1504         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1505                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1506
1507                 switch (items->type) {
1508                 case RTE_FLOW_ITEM_TYPE_VOID:
1509                         break;
1510                 case RTE_FLOW_ITEM_TYPE_ETH:
1511                         flow_verbs_translate_item_eth(dev_flow, items,
1512                                                       item_flags);
1513                         subpriority = MLX5_PRIORITY_MAP_L2;
1514                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1515                                                MLX5_FLOW_LAYER_OUTER_L2;
1516                         break;
1517                 case RTE_FLOW_ITEM_TYPE_VLAN:
1518                         flow_verbs_translate_item_vlan(dev_flow, items,
1519                                                        item_flags);
1520                         subpriority = MLX5_PRIORITY_MAP_L2;
1521                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1522                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1523                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1524                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1525                         break;
1526                 case RTE_FLOW_ITEM_TYPE_IPV4:
1527                         flow_verbs_translate_item_ipv4(dev_flow, items,
1528                                                        item_flags);
1529                         subpriority = MLX5_PRIORITY_MAP_L3;
1530                         dev_flow->hash_fields |=
1531                                 mlx5_flow_hashfields_adjust
1532                                         (dev_flow, tunnel,
1533                                          MLX5_IPV4_LAYER_TYPES,
1534                                          MLX5_IPV4_IBV_RX_HASH);
1535                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1536                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1537                         break;
1538                 case RTE_FLOW_ITEM_TYPE_IPV6:
1539                         flow_verbs_translate_item_ipv6(dev_flow, items,
1540                                                        item_flags);
1541                         subpriority = MLX5_PRIORITY_MAP_L3;
1542                         dev_flow->hash_fields |=
1543                                 mlx5_flow_hashfields_adjust
1544                                         (dev_flow, tunnel,
1545                                          MLX5_IPV6_LAYER_TYPES,
1546                                          MLX5_IPV6_IBV_RX_HASH);
1547                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1548                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1549                         break;
1550                 case RTE_FLOW_ITEM_TYPE_TCP:
1551                         flow_verbs_translate_item_tcp(dev_flow, items,
1552                                                       item_flags);
1553                         subpriority = MLX5_PRIORITY_MAP_L4;
1554                         dev_flow->hash_fields |=
1555                                 mlx5_flow_hashfields_adjust
1556                                         (dev_flow, tunnel, ETH_RSS_TCP,
1557                                          (IBV_RX_HASH_SRC_PORT_TCP |
1558                                           IBV_RX_HASH_DST_PORT_TCP));
1559                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1560                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1561                         break;
1562                 case RTE_FLOW_ITEM_TYPE_UDP:
1563                         flow_verbs_translate_item_udp(dev_flow, items,
1564                                                       item_flags);
1565                         subpriority = MLX5_PRIORITY_MAP_L4;
1566                         dev_flow->hash_fields |=
1567                                 mlx5_flow_hashfields_adjust
1568                                         (dev_flow, tunnel, ETH_RSS_UDP,
1569                                          (IBV_RX_HASH_SRC_PORT_UDP |
1570                                           IBV_RX_HASH_DST_PORT_UDP));
1571                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1572                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1573                         break;
1574                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1575                         flow_verbs_translate_item_vxlan(dev_flow, items,
1576                                                         item_flags);
1577                         subpriority = MLX5_PRIORITY_MAP_L2;
1578                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1579                         break;
1580                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1581                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1582                                                             item_flags);
1583                         subpriority = MLX5_PRIORITY_MAP_L2;
1584                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1585                         break;
1586                 case RTE_FLOW_ITEM_TYPE_GRE:
1587                         flow_verbs_translate_item_gre(dev_flow, items,
1588                                                       item_flags);
1589                         subpriority = MLX5_PRIORITY_MAP_L2;
1590                         item_flags |= MLX5_FLOW_LAYER_GRE;
1591                         break;
1592                 case RTE_FLOW_ITEM_TYPE_MPLS:
1593                         flow_verbs_translate_item_mpls(dev_flow, items,
1594                                                        item_flags);
1595                         subpriority = MLX5_PRIORITY_MAP_L2;
1596                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1597                         break;
1598                 default:
1599                         return rte_flow_error_set(error, ENOTSUP,
1600                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1601                                                   NULL,
1602                                                   "item not supported");
1603                 }
1604         }
1605         dev_flow->layers = item_flags;
1606         dev_flow->verbs.attr->priority =
1607                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1608         dev_flow->verbs.attr->port = (uint8_t)priv->ibv_port;
1609         return 0;
1610 }
1611
1612 /**
1613  * Remove the flow from the NIC but keeps it in memory.
1614  *
1615  * @param[in] dev
1616  *   Pointer to the Ethernet device structure.
1617  * @param[in, out] flow
1618  *   Pointer to flow structure.
1619  */
1620 static void
1621 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1622 {
1623         struct mlx5_flow_verbs *verbs;
1624         struct mlx5_flow *dev_flow;
1625
1626         if (!flow)
1627                 return;
1628         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1629                 verbs = &dev_flow->verbs;
1630                 if (verbs->flow) {
1631                         claim_zero(mlx5_glue->destroy_flow(verbs->flow));
1632                         verbs->flow = NULL;
1633                 }
1634                 if (verbs->hrxq) {
1635                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
1636                                 mlx5_hrxq_drop_release(dev);
1637                         else
1638                                 mlx5_hrxq_release(dev, verbs->hrxq);
1639                         verbs->hrxq = NULL;
1640                 }
1641                 if (dev_flow->verbs.vf_vlan.tag &&
1642                     dev_flow->verbs.vf_vlan.created) {
1643                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1644                 }
1645         }
1646 }
1647
1648 /**
1649  * Remove the flow from the NIC and the memory.
1650  *
1651  * @param[in] dev
1652  *   Pointer to the Ethernet device structure.
1653  * @param[in, out] flow
1654  *   Pointer to flow structure.
1655  */
1656 static void
1657 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1658 {
1659         struct mlx5_flow *dev_flow;
1660
1661         if (!flow)
1662                 return;
1663         flow_verbs_remove(dev, flow);
1664         while (!LIST_EMPTY(&flow->dev_flows)) {
1665                 dev_flow = LIST_FIRST(&flow->dev_flows);
1666                 LIST_REMOVE(dev_flow, next);
1667                 rte_free(dev_flow);
1668         }
1669         if (flow->counter) {
1670                 flow_verbs_counter_release(dev, flow->counter);
1671                 flow->counter = NULL;
1672         }
1673 }
1674
1675 /**
1676  * Apply the flow to the NIC.
1677  *
1678  * @param[in] dev
1679  *   Pointer to the Ethernet device structure.
1680  * @param[in, out] flow
1681  *   Pointer to flow structure.
1682  * @param[out] error
1683  *   Pointer to error structure.
1684  *
1685  * @return
1686  *   0 on success, a negative errno value otherwise and rte_errno is set.
1687  */
1688 static int
1689 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1690                  struct rte_flow_error *error)
1691 {
1692         struct mlx5_priv *priv = dev->data->dev_private;
1693         struct mlx5_flow_verbs *verbs;
1694         struct mlx5_flow *dev_flow;
1695         int err;
1696
1697         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1698                 verbs = &dev_flow->verbs;
1699                 if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) {
1700                         verbs->hrxq = mlx5_hrxq_drop_new(dev);
1701                         if (!verbs->hrxq) {
1702                                 rte_flow_error_set
1703                                         (error, errno,
1704                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1705                                          "cannot get drop hash queue");
1706                                 goto error;
1707                         }
1708                 } else {
1709                         struct mlx5_hrxq *hrxq;
1710
1711                         assert(flow->rss.queue);
1712                         hrxq = mlx5_hrxq_get(dev, flow->rss.key,
1713                                              MLX5_RSS_HASH_KEY_LEN,
1714                                              dev_flow->hash_fields,
1715                                              (*flow->rss.queue),
1716                                              flow->rss.queue_num);
1717                         if (!hrxq)
1718                                 hrxq = mlx5_hrxq_new(dev, flow->rss.key,
1719                                                      MLX5_RSS_HASH_KEY_LEN,
1720                                                      dev_flow->hash_fields,
1721                                                      (*flow->rss.queue),
1722                                                      flow->rss.queue_num,
1723                                                      !!(dev_flow->layers &
1724                                                        MLX5_FLOW_LAYER_TUNNEL));
1725                         if (!hrxq) {
1726                                 rte_flow_error_set
1727                                         (error, rte_errno,
1728                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1729                                          "cannot get hash queue");
1730                                 goto error;
1731                         }
1732                         verbs->hrxq = hrxq;
1733                 }
1734                 verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp,
1735                                                      verbs->attr);
1736                 if (!verbs->flow) {
1737                         rte_flow_error_set(error, errno,
1738                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1739                                            NULL,
1740                                            "hardware refuses to create flow");
1741                         goto error;
1742                 }
1743                 if (priv->vmwa_context &&
1744                     dev_flow->verbs.vf_vlan.tag &&
1745                     !dev_flow->verbs.vf_vlan.created) {
1746                         /*
1747                          * The rule contains the VLAN pattern.
1748                          * For VF we are going to create VLAN
1749                          * interface to make hypervisor set correct
1750                          * e-Switch vport context.
1751                          */
1752                         mlx5_vlan_vmwa_acquire(dev, &dev_flow->verbs.vf_vlan);
1753                 }
1754         }
1755         return 0;
1756 error:
1757         err = rte_errno; /* Save rte_errno before cleanup. */
1758         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1759                 verbs = &dev_flow->verbs;
1760                 if (verbs->hrxq) {
1761                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
1762                                 mlx5_hrxq_drop_release(dev);
1763                         else
1764                                 mlx5_hrxq_release(dev, verbs->hrxq);
1765                         verbs->hrxq = NULL;
1766                 }
1767                 if (dev_flow->verbs.vf_vlan.tag &&
1768                     dev_flow->verbs.vf_vlan.created) {
1769                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1770                 }
1771         }
1772         rte_errno = err; /* Restore rte_errno. */
1773         return -rte_errno;
1774 }
1775
1776 /**
1777  * Query a flow.
1778  *
1779  * @see rte_flow_query()
1780  * @see rte_flow_ops
1781  */
1782 static int
1783 flow_verbs_query(struct rte_eth_dev *dev,
1784                  struct rte_flow *flow,
1785                  const struct rte_flow_action *actions,
1786                  void *data,
1787                  struct rte_flow_error *error)
1788 {
1789         int ret = -EINVAL;
1790
1791         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1792                 switch (actions->type) {
1793                 case RTE_FLOW_ACTION_TYPE_VOID:
1794                         break;
1795                 case RTE_FLOW_ACTION_TYPE_COUNT:
1796                         ret = flow_verbs_counter_query(dev, flow, data, error);
1797                         break;
1798                 default:
1799                         return rte_flow_error_set(error, ENOTSUP,
1800                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1801                                                   actions,
1802                                                   "action not supported");
1803                 }
1804         }
1805         return ret;
1806 }
1807
1808 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1809         .validate = flow_verbs_validate,
1810         .prepare = flow_verbs_prepare,
1811         .translate = flow_verbs_translate,
1812         .apply = flow_verbs_apply,
1813         .remove = flow_verbs_remove,
1814         .destroy = flow_verbs_destroy,
1815         .query = flow_verbs_query,
1816 };