drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28
  29 #include "mlx5.h"
  30 #include "mlx5_defs.h"
  31 #include "mlx5_flow.h"
  32 #include "mlx5_glue.h"
  33 #include "mlx5_prm.h"
  34 #include "mlx5_rxtx.h"
  35
  36 #define VERBS_SPEC_INNER(item_flags) \
  37         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  38
  39 /**
  40  * Create Verbs flow counter with Verbs library.
  41  *
  42  * @param[in] dev
  43  *   Pointer to the Ethernet device structure.
  44  * @param[in, out] counter
  45  *   mlx5 flow counter object, contains the counter id,
  46  *   handle of created Verbs flow counter is returned
  47  *   in cs field (if counters are supported).
  48  *
  49  * @return
  50  *   0 On success else a negative errno value is returned
  51  *   and rte_errno is set.
  52  */
  53 static int
  54 flow_verbs_counter_create(struct rte_eth_dev *dev,
  55                           struct mlx5_flow_counter *counter)
  56 {
  57 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  58         struct mlx5_priv *priv = dev->data->dev_private;
  59         struct ibv_context *ctx = priv->sh->ctx;
  60         struct ibv_counter_set_init_attr init = {
  61                          .counter_set_id = counter->id};
  62
  63         counter->cs = mlx5_glue->create_counter_set(ctx, &init);
  64         if (!counter->cs) {
  65                 rte_errno = ENOTSUP;
  66                 return -ENOTSUP;
  67         }
  68         return 0;
  69 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
  70         struct mlx5_priv *priv = dev->data->dev_private;
  71         struct ibv_context *ctx = priv->sh->ctx;
  72         struct ibv_counters_init_attr init = {0};
  73         struct ibv_counter_attach_attr attach;
  74         int ret;
  75
  76         memset(&attach, 0, sizeof(attach));
  77         counter->cs = mlx5_glue->create_counters(ctx, &init);
  78         if (!counter->cs) {
  79                 rte_errno = ENOTSUP;
  80                 return -ENOTSUP;
  81         }
  82         attach.counter_desc = IBV_COUNTER_PACKETS;
  83         attach.index = 0;
  84         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
  85         if (!ret) {
  86                 attach.counter_desc = IBV_COUNTER_BYTES;
  87                 attach.index = 1;
  88                 ret = mlx5_glue->attach_counters
  89                                         (counter->cs, &attach, NULL);
  90         }
  91         if (ret) {
  92                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
  93                 counter->cs = NULL;
  94                 rte_errno = ret;
  95                 return -ret;
  96         }
  97         return 0;
  98 #else
  99         (void)dev;
 100         (void)counter;
 101         rte_errno = ENOTSUP;
 102         return -ENOTSUP;
 103 #endif
 104 }
 105
 106 /**
 107  * Get a flow counter.
 108  *
 109  * @param[in] dev
 110  *   Pointer to the Ethernet device structure.
 111  * @param[in] shared
 112  *   Indicate if this counter is shared with other flows.
 113  * @param[in] id
 114  *   Counter identifier.
 115  *
 116  * @return
 117  *   A pointer to the counter, NULL otherwise and rte_errno is set.
 118  */
 119 static struct mlx5_flow_counter *
 120 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 121 {
 122         struct mlx5_priv *priv = dev->data->dev_private;
 123         struct mlx5_flow_counter *cnt;
 124         int ret;
 125
 126         if (shared) {
 127                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
 128                         if (cnt->shared && cnt->id == id) {
 129                                 cnt->ref_cnt++;
 130                                 return cnt;
 131                         }
 132                 }
 133         }
 134         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 135         if (!cnt) {
 136                 rte_errno = ENOMEM;
 137                 return NULL;
 138         }
 139         cnt->id = id;
 140         cnt->shared = shared;
 141         cnt->ref_cnt = 1;
 142         cnt->hits = 0;
 143         cnt->bytes = 0;
 144         /* Create counter with Verbs. */
 145         ret = flow_verbs_counter_create(dev, cnt);
 146         if (!ret) {
 147                 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
 148                 return cnt;
 149         }
 150         /* Some error occurred in Verbs library. */
 151         rte_free(cnt);
 152         rte_errno = -ret;
 153         return NULL;
 154 }
 155
 156 /**
 157  * Release a flow counter.
 158  *
 159  * @param[in] dev
 160  *   Pointer to the Ethernet device structure.
 161  * @param[in] counter
 162  *   Pointer to the counter handler.
 163  */
 164 static void
 165 flow_verbs_counter_release(struct rte_eth_dev *dev,
 166                            struct mlx5_flow_counter *counter)
 167 {
 168         struct mlx5_priv *priv = dev->data->dev_private;
 169
 170         if (--counter->ref_cnt == 0) {
 171 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 172                 claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
 173 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 174                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 175 #endif
 176                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
 177                 rte_free(counter);
 178         }
 179 }
 180
 181 /**
 182  * Query a flow counter via Verbs library call.
 183  *
 184  * @see rte_flow_query()
 185  * @see rte_flow_ops
 186  */
 187 static int
 188 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 189                          struct rte_flow *flow, void *data,
 190                          struct rte_flow_error *error)
 191 {
 192 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 193         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 194         if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 195                 struct rte_flow_query_count *qc = data;
 196                 uint64_t counters[2] = {0, 0};
 197 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 198                 struct ibv_query_counter_set_attr query_cs_attr = {
 199                         .cs = flow->counter->cs,
 200                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 201                 };
 202                 struct ibv_counter_set_data query_out = {
 203                         .out = counters,
 204                         .outlen = 2 * sizeof(uint64_t),
 205                 };
 206                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 207                                                        &query_out);
 208 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 209                 int err = mlx5_glue->query_counters
 210                                (flow->counter->cs, counters,
 211                                 RTE_DIM(counters),
 212                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 213 #endif
 214                 if (err)
 215                         return rte_flow_error_set
 216                                 (error, err,
 217                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 218                                  NULL,
 219                                  "cannot read counter");
 220                 qc->hits_set = 1;
 221                 qc->bytes_set = 1;
 222                 qc->hits = counters[0] - flow->counter->hits;
 223                 qc->bytes = counters[1] - flow->counter->bytes;
 224                 if (qc->reset) {
 225                         flow->counter->hits = counters[0];
 226                         flow->counter->bytes = counters[1];
 227                 }
 228                 return 0;
 229         }
 230         return rte_flow_error_set(error, EINVAL,
 231                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 232                                   NULL,
 233                                   "flow does not have counter");
 234 #else
 235         (void)flow;
 236         (void)data;
 237         return rte_flow_error_set(error, ENOTSUP,
 238                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 239                                   NULL,
 240                                   "counters are not available");
 241 #endif
 242 }
 243
 244 /**
 245  * Add a verbs item specification into @p verbs.
 246  *
 247  * @param[out] verbs
 248  *   Pointer to verbs structure.
 249  * @param[in] src
 250  *   Create specification.
 251  * @param[in] size
 252  *   Size in bytes of the specification to copy.
 253  */
 254 static void
 255 flow_verbs_spec_add(struct mlx5_flow_verbs *verbs, void *src, unsigned int size)
 256 {
 257         void *dst;
 258
 259         if (!verbs)
 260                 return;
 261         assert(verbs->specs);
 262         dst = (void *)(verbs->specs + verbs->size);
 263         memcpy(dst, src, size);
 264         ++verbs->attr->num_of_specs;
 265         verbs->size += size;
 266 }
 267
 268 /**
 269  * Convert the @p item into a Verbs specification. This function assumes that
 270  * the input is valid and that there is space to insert the requested item
 271  * into the flow.
 272  *
 273  * @param[in, out] dev_flow
 274  *   Pointer to dev_flow structure.
 275  * @param[in] item
 276  *   Item specification.
 277  * @param[in] item_flags
 278  *   Parsed item flags.
 279  */
 280 static void
 281 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 282                               const struct rte_flow_item *item,
 283                               uint64_t item_flags)
 284 {
 285         const struct rte_flow_item_eth *spec = item->spec;
 286         const struct rte_flow_item_eth *mask = item->mask;
 287         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 288         struct ibv_flow_spec_eth eth = {
 289                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 290                 .size = size,
 291         };
 292
 293         if (!mask)
 294                 mask = &rte_flow_item_eth_mask;
 295         if (spec) {
 296                 unsigned int i;
 297
 298                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 299                         RTE_ETHER_ADDR_LEN);
 300                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 301                         RTE_ETHER_ADDR_LEN);
 302                 eth.val.ether_type = spec->type;
 303                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 304                         RTE_ETHER_ADDR_LEN);
 305                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 306                         RTE_ETHER_ADDR_LEN);
 307                 eth.mask.ether_type = mask->type;
 308                 /* Remove unwanted bits from values. */
 309                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 310                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 311                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 312                 }
 313                 eth.val.ether_type &= eth.mask.ether_type;
 314         }
 315         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 316 }
 317
 318 /**
 319  * Update the VLAN tag in the Verbs Ethernet specification.
 320  * This function assumes that the input is valid and there is space to add
 321  * the requested item.
 322  *
 323  * @param[in, out] attr
 324  *   Pointer to Verbs attributes structure.
 325  * @param[in] eth
 326  *   Verbs structure containing the VLAN information to copy.
 327  */
 328 static void
 329 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 330                             struct ibv_flow_spec_eth *eth)
 331 {
 332         unsigned int i;
 333         const enum ibv_flow_spec_type search = eth->type;
 334         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 335                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 336
 337         for (i = 0; i != attr->num_of_specs; ++i) {
 338                 if (hdr->type == search) {
 339                         struct ibv_flow_spec_eth *e =
 340                                 (struct ibv_flow_spec_eth *)hdr;
 341
 342                         e->val.vlan_tag = eth->val.vlan_tag;
 343                         e->mask.vlan_tag = eth->mask.vlan_tag;
 344                         e->val.ether_type = eth->val.ether_type;
 345                         e->mask.ether_type = eth->mask.ether_type;
 346                         break;
 347                 }
 348                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 349         }
 350 }
 351
 352 /**
 353  * Convert the @p item into a Verbs specification. This function assumes that
 354  * the input is valid and that there is space to insert the requested item
 355  * into the flow.
 356  *
 357  * @param[in, out] dev_flow
 358  *   Pointer to dev_flow structure.
 359  * @param[in] item
 360  *   Item specification.
 361  * @param[in] item_flags
 362  *   Parsed item flags.
 363  */
 364 static void
 365 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 366                                const struct rte_flow_item *item,
 367                                uint64_t item_flags)
 368 {
 369         const struct rte_flow_item_vlan *spec = item->spec;
 370         const struct rte_flow_item_vlan *mask = item->mask;
 371         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 372         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 373         struct ibv_flow_spec_eth eth = {
 374                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 375                 .size = size,
 376         };
 377         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 378                                       MLX5_FLOW_LAYER_OUTER_L2;
 379
 380         if (!mask)
 381                 mask = &rte_flow_item_vlan_mask;
 382         if (spec) {
 383                 eth.val.vlan_tag = spec->tci;
 384                 eth.mask.vlan_tag = mask->tci;
 385                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 386                 eth.val.ether_type = spec->inner_type;
 387                 eth.mask.ether_type = mask->inner_type;
 388                 eth.val.ether_type &= eth.mask.ether_type;
 389         }
 390         if (!(item_flags & l2m))
 391                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 392         else
 393                 flow_verbs_item_vlan_update(dev_flow->verbs.attr, &eth);
 394         if (!tunnel)
 395                 dev_flow->verbs.vf_vlan.tag =
 396                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 397 }
 398
 399 /**
 400  * Convert the @p item into a Verbs specification. This function assumes that
 401  * the input is valid and that there is space to insert the requested item
 402  * into the flow.
 403  *
 404  * @param[in, out] dev_flow
 405  *   Pointer to dev_flow structure.
 406  * @param[in] item
 407  *   Item specification.
 408  * @param[in] item_flags
 409  *   Parsed item flags.
 410  */
 411 static void
 412 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 413                                const struct rte_flow_item *item,
 414                                uint64_t item_flags)
 415 {
 416         const struct rte_flow_item_ipv4 *spec = item->spec;
 417         const struct rte_flow_item_ipv4 *mask = item->mask;
 418         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 419         struct ibv_flow_spec_ipv4_ext ipv4 = {
 420                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 421                 .size = size,
 422         };
 423
 424         if (!mask)
 425                 mask = &rte_flow_item_ipv4_mask;
 426         if (spec) {
 427                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 428                         .src_ip = spec->hdr.src_addr,
 429                         .dst_ip = spec->hdr.dst_addr,
 430                         .proto = spec->hdr.next_proto_id,
 431                         .tos = spec->hdr.type_of_service,
 432                 };
 433                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 434                         .src_ip = mask->hdr.src_addr,
 435                         .dst_ip = mask->hdr.dst_addr,
 436                         .proto = mask->hdr.next_proto_id,
 437                         .tos = mask->hdr.type_of_service,
 438                 };
 439                 /* Remove unwanted bits from values. */
 440                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 441                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 442                 ipv4.val.proto &= ipv4.mask.proto;
 443                 ipv4.val.tos &= ipv4.mask.tos;
 444         }
 445         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 446 }
 447
 448 /**
 449  * Convert the @p item into a Verbs specification. This function assumes that
 450  * the input is valid and that there is space to insert the requested item
 451  * into the flow.
 452  *
 453  * @param[in, out] dev_flow
 454  *   Pointer to dev_flow structure.
 455  * @param[in] item
 456  *   Item specification.
 457  * @param[in] item_flags
 458  *   Parsed item flags.
 459  */
 460 static void
 461 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 462                                const struct rte_flow_item *item,
 463                                uint64_t item_flags)
 464 {
 465         const struct rte_flow_item_ipv6 *spec = item->spec;
 466         const struct rte_flow_item_ipv6 *mask = item->mask;
 467         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 468         struct ibv_flow_spec_ipv6 ipv6 = {
 469                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 470                 .size = size,
 471         };
 472
 473         if (!mask)
 474                 mask = &rte_flow_item_ipv6_mask;
 475         if (spec) {
 476                 unsigned int i;
 477                 uint32_t vtc_flow_val;
 478                 uint32_t vtc_flow_mask;
 479
 480                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 481                        RTE_DIM(ipv6.val.src_ip));
 482                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 483                        RTE_DIM(ipv6.val.dst_ip));
 484                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 485                        RTE_DIM(ipv6.mask.src_ip));
 486                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 487                        RTE_DIM(ipv6.mask.dst_ip));
 488                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 489                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 490                 ipv6.val.flow_label =
 491                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 492                                          RTE_IPV6_HDR_FL_SHIFT);
 493                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 494                                          RTE_IPV6_HDR_TC_SHIFT;
 495                 ipv6.val.next_hdr = spec->hdr.proto;
 496                 ipv6.val.hop_limit = spec->hdr.hop_limits;
 497                 ipv6.mask.flow_label =
 498                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 499                                          RTE_IPV6_HDR_FL_SHIFT);
 500                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 501                                           RTE_IPV6_HDR_TC_SHIFT;
 502                 ipv6.mask.next_hdr = mask->hdr.proto;
 503                 ipv6.mask.hop_limit = mask->hdr.hop_limits;
 504                 /* Remove unwanted bits from values. */
 505                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 506                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 507                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 508                 }
 509                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 510                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 511                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 512                 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 513         }
 514         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 515 }
 516
 517 /**
 518  * Convert the @p item into a Verbs specification. This function assumes that
 519  * the input is valid and that there is space to insert the requested item
 520  * into the flow.
 521  *
 522  * @param[in, out] dev_flow
 523  *   Pointer to dev_flow structure.
 524  * @param[in] item
 525  *   Item specification.
 526  * @param[in] item_flags
 527  *   Parsed item flags.
 528  */
 529 static void
 530 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 531                               const struct rte_flow_item *item,
 532                               uint64_t item_flags __rte_unused)
 533 {
 534         const struct rte_flow_item_tcp *spec = item->spec;
 535         const struct rte_flow_item_tcp *mask = item->mask;
 536         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 537         struct ibv_flow_spec_tcp_udp tcp = {
 538                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 539                 .size = size,
 540         };
 541
 542         if (!mask)
 543                 mask = &rte_flow_item_tcp_mask;
 544         if (spec) {
 545                 tcp.val.dst_port = spec->hdr.dst_port;
 546                 tcp.val.src_port = spec->hdr.src_port;
 547                 tcp.mask.dst_port = mask->hdr.dst_port;
 548                 tcp.mask.src_port = mask->hdr.src_port;
 549                 /* Remove unwanted bits from values. */
 550                 tcp.val.src_port &= tcp.mask.src_port;
 551                 tcp.val.dst_port &= tcp.mask.dst_port;
 552         }
 553         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 554 }
 555
 556 /**
 557  * Convert the @p item into a Verbs specification. This function assumes that
 558  * the input is valid and that there is space to insert the requested item
 559  * into the flow.
 560  *
 561  * @param[in, out] dev_flow
 562  *   Pointer to dev_flow structure.
 563  * @param[in] item
 564  *   Item specification.
 565  * @param[in] item_flags
 566  *   Parsed item flags.
 567  */
 568 static void
 569 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 570                               const struct rte_flow_item *item,
 571                               uint64_t item_flags __rte_unused)
 572 {
 573         const struct rte_flow_item_udp *spec = item->spec;
 574         const struct rte_flow_item_udp *mask = item->mask;
 575         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 576         struct ibv_flow_spec_tcp_udp udp = {
 577                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 578                 .size = size,
 579         };
 580
 581         if (!mask)
 582                 mask = &rte_flow_item_udp_mask;
 583         if (spec) {
 584                 udp.val.dst_port = spec->hdr.dst_port;
 585                 udp.val.src_port = spec->hdr.src_port;
 586                 udp.mask.dst_port = mask->hdr.dst_port;
 587                 udp.mask.src_port = mask->hdr.src_port;
 588                 /* Remove unwanted bits from values. */
 589                 udp.val.src_port &= udp.mask.src_port;
 590                 udp.val.dst_port &= udp.mask.dst_port;
 591         }
 592         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 593 }
 594
 595 /**
 596  * Convert the @p item into a Verbs specification. This function assumes that
 597  * the input is valid and that there is space to insert the requested item
 598  * into the flow.
 599  *
 600  * @param[in, out] dev_flow
 601  *   Pointer to dev_flow structure.
 602  * @param[in] item
 603  *   Item specification.
 604  * @param[in] item_flags
 605  *   Parsed item flags.
 606  */
 607 static void
 608 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 609                                 const struct rte_flow_item *item,
 610                                 uint64_t item_flags __rte_unused)
 611 {
 612         const struct rte_flow_item_vxlan *spec = item->spec;
 613         const struct rte_flow_item_vxlan *mask = item->mask;
 614         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 615         struct ibv_flow_spec_tunnel vxlan = {
 616                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 617                 .size = size,
 618         };
 619         union vni {
 620                 uint32_t vlan_id;
 621                 uint8_t vni[4];
 622         } id = { .vlan_id = 0, };
 623
 624         if (!mask)
 625                 mask = &rte_flow_item_vxlan_mask;
 626         if (spec) {
 627                 memcpy(&id.vni[1], spec->vni, 3);
 628                 vxlan.val.tunnel_id = id.vlan_id;
 629                 memcpy(&id.vni[1], mask->vni, 3);
 630                 vxlan.mask.tunnel_id = id.vlan_id;
 631                 /* Remove unwanted bits from values. */
 632                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 633         }
 634         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 635 }
 636
 637 /**
 638  * Convert the @p item into a Verbs specification. This function assumes that
 639  * the input is valid and that there is space to insert the requested item
 640  * into the flow.
 641  *
 642  * @param[in, out] dev_flow
 643  *   Pointer to dev_flow structure.
 644  * @param[in] item
 645  *   Item specification.
 646  * @param[in] item_flags
 647  *   Parsed item flags.
 648  */
 649 static void
 650 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 651                                     const struct rte_flow_item *item,
 652                                     uint64_t item_flags __rte_unused)
 653 {
 654         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 655         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 656         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 657         struct ibv_flow_spec_tunnel vxlan_gpe = {
 658                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 659                 .size = size,
 660         };
 661         union vni {
 662                 uint32_t vlan_id;
 663                 uint8_t vni[4];
 664         } id = { .vlan_id = 0, };
 665
 666         if (!mask)
 667                 mask = &rte_flow_item_vxlan_gpe_mask;
 668         if (spec) {
 669                 memcpy(&id.vni[1], spec->vni, 3);
 670                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 671                 memcpy(&id.vni[1], mask->vni, 3);
 672                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 673                 /* Remove unwanted bits from values. */
 674                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 675         }
 676         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 677 }
 678
 679 /**
 680  * Update the protocol in Verbs IPv4/IPv6 spec.
 681  *
 682  * @param[in, out] attr
 683  *   Pointer to Verbs attributes structure.
 684  * @param[in] search
 685  *   Specification type to search in order to update the IP protocol.
 686  * @param[in] protocol
 687  *   Protocol value to set if none is present in the specification.
 688  */
 689 static void
 690 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 691                                        enum ibv_flow_spec_type search,
 692                                        uint8_t protocol)
 693 {
 694         unsigned int i;
 695         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 696                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 697
 698         if (!attr)
 699                 return;
 700         for (i = 0; i != attr->num_of_specs; ++i) {
 701                 if (hdr->type == search) {
 702                         union {
 703                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 704                                 struct ibv_flow_spec_ipv6 *ipv6;
 705                         } ip;
 706
 707                         switch (search) {
 708                         case IBV_FLOW_SPEC_IPV4_EXT:
 709                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 710                                 if (!ip.ipv4->val.proto) {
 711                                         ip.ipv4->val.proto = protocol;
 712                                         ip.ipv4->mask.proto = 0xff;
 713                                 }
 714                                 break;
 715                         case IBV_FLOW_SPEC_IPV6:
 716                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 717                                 if (!ip.ipv6->val.next_hdr) {
 718                                         ip.ipv6->val.next_hdr = protocol;
 719                                         ip.ipv6->mask.next_hdr = 0xff;
 720                                 }
 721                                 break;
 722                         default:
 723                                 break;
 724                         }
 725                         break;
 726                 }
 727                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 728         }
 729 }
 730
 731 /**
 732  * Convert the @p item into a Verbs specification. This function assumes that
 733  * the input is valid and that there is space to insert the requested item
 734  * into the flow.
 735  *
 736  * @param[in, out] dev_flow
 737  *   Pointer to dev_flow structure.
 738  * @param[in] item
 739  *   Item specification.
 740  * @param[in] item_flags
 741  *   Parsed item flags.
 742  */
 743 static void
 744 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 745                               const struct rte_flow_item *item __rte_unused,
 746                               uint64_t item_flags)
 747 {
 748         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
 749 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 750         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 751         struct ibv_flow_spec_tunnel tunnel = {
 752                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 753                 .size = size,
 754         };
 755 #else
 756         const struct rte_flow_item_gre *spec = item->spec;
 757         const struct rte_flow_item_gre *mask = item->mask;
 758         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 759         struct ibv_flow_spec_gre tunnel = {
 760                 .type = IBV_FLOW_SPEC_GRE,
 761                 .size = size,
 762         };
 763
 764         if (!mask)
 765                 mask = &rte_flow_item_gre_mask;
 766         if (spec) {
 767                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 768                 tunnel.val.protocol = spec->protocol;
 769                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 770                 tunnel.mask.protocol = mask->protocol;
 771                 /* Remove unwanted bits from values. */
 772                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 773                 tunnel.val.protocol &= tunnel.mask.protocol;
 774                 tunnel.val.key &= tunnel.mask.key;
 775         }
 776 #endif
 777         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 778                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 779                                                        IBV_FLOW_SPEC_IPV4_EXT,
 780                                                        IPPROTO_GRE);
 781         else
 782                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 783                                                        IBV_FLOW_SPEC_IPV6,
 784                                                        IPPROTO_GRE);
 785         flow_verbs_spec_add(verbs, &tunnel, size);
 786 }
 787
 788 /**
 789  * Convert the @p action into a Verbs specification. This function assumes that
 790  * the input is valid and that there is space to insert the requested action
 791  * into the flow. This function also return the action that was added.
 792  *
 793  * @param[in, out] dev_flow
 794  *   Pointer to dev_flow structure.
 795  * @param[in] item
 796  *   Item specification.
 797  * @param[in] item_flags
 798  *   Parsed item flags.
 799  */
 800 static void
 801 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
 802                                const struct rte_flow_item *item __rte_unused,
 803                                uint64_t item_flags __rte_unused)
 804 {
 805 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 806         const struct rte_flow_item_mpls *spec = item->spec;
 807         const struct rte_flow_item_mpls *mask = item->mask;
 808         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 809         struct ibv_flow_spec_mpls mpls = {
 810                 .type = IBV_FLOW_SPEC_MPLS,
 811                 .size = size,
 812         };
 813
 814         if (!mask)
 815                 mask = &rte_flow_item_mpls_mask;
 816         if (spec) {
 817                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 818                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 819                 /* Remove unwanted bits from values.  */
 820                 mpls.val.label &= mpls.mask.label;
 821         }
 822         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
 823 #endif
 824 }
 825
 826 /**
 827  * Convert the @p action into a Verbs specification. This function assumes that
 828  * the input is valid and that there is space to insert the requested action
 829  * into the flow.
 830  *
 831  * @param[in] dev_flow
 832  *   Pointer to mlx5_flow.
 833  * @param[in] action
 834  *   Action configuration.
 835  */
 836 static void
 837 flow_verbs_translate_action_drop
 838         (struct mlx5_flow *dev_flow,
 839          const struct rte_flow_action *action __rte_unused)
 840 {
 841         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 842         struct ibv_flow_spec_action_drop drop = {
 843                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 844                         .size = size,
 845         };
 846
 847         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
 848 }
 849
 850 /**
 851  * Convert the @p action into a Verbs specification. This function assumes that
 852  * the input is valid and that there is space to insert the requested action
 853  * into the flow.
 854  *
 855  * @param[in] dev_flow
 856  *   Pointer to mlx5_flow.
 857  * @param[in] action
 858  *   Action configuration.
 859  */
 860 static void
 861 flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow,
 862                                   const struct rte_flow_action *action)
 863 {
 864         const struct rte_flow_action_queue *queue = action->conf;
 865         struct rte_flow *flow = dev_flow->flow;
 866
 867         if (flow->queue)
 868                 (*flow->queue)[0] = queue->index;
 869         flow->rss.queue_num = 1;
 870 }
 871
 872 /**
 873  * Convert the @p action into a Verbs specification. This function assumes that
 874  * the input is valid and that there is space to insert the requested action
 875  * into the flow.
 876  *
 877  * @param[in] action
 878  *   Action configuration.
 879  * @param[in, out] action_flags
 880  *   Pointer to the detected actions.
 881  * @param[in] dev_flow
 882  *   Pointer to mlx5_flow.
 883  */
 884 static void
 885 flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow,
 886                                 const struct rte_flow_action *action)
 887 {
 888         const struct rte_flow_action_rss *rss = action->conf;
 889         const uint8_t *rss_key;
 890         struct rte_flow *flow = dev_flow->flow;
 891
 892         if (flow->queue)
 893                 memcpy((*flow->queue), rss->queue,
 894                        rss->queue_num * sizeof(uint16_t));
 895         flow->rss.queue_num = rss->queue_num;
 896         /* NULL RSS key indicates default RSS key. */
 897         rss_key = !rss->key ? rss_hash_default_key : rss->key;
 898         memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
 899         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
 900         flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
 901         flow->rss.level = rss->level;
 902 }
 903
 904 /**
 905  * Convert the @p action into a Verbs specification. This function assumes that
 906  * the input is valid and that there is space to insert the requested action
 907  * into the flow.
 908  *
 909  * @param[in] dev_flow
 910  *   Pointer to mlx5_flow.
 911  * @param[in] action
 912  *   Action configuration.
 913  */
 914 static void
 915 flow_verbs_translate_action_flag
 916         (struct mlx5_flow *dev_flow,
 917          const struct rte_flow_action *action __rte_unused)
 918 {
 919         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 920         struct ibv_flow_spec_action_tag tag = {
 921                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 922                 .size = size,
 923                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 924         };
 925
 926         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 927 }
 928
 929 /**
 930  * Convert the @p action into a Verbs specification. This function assumes that
 931  * the input is valid and that there is space to insert the requested action
 932  * into the flow.
 933  *
 934  * @param[in] dev_flow
 935  *   Pointer to mlx5_flow.
 936  * @param[in] action
 937  *   Action configuration.
 938  */
 939 static void
 940 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
 941                                  const struct rte_flow_action *action)
 942 {
 943         const struct rte_flow_action_mark *mark = action->conf;
 944         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 945         struct ibv_flow_spec_action_tag tag = {
 946                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 947                 .size = size,
 948                 .tag_id = mlx5_flow_mark_set(mark->id),
 949         };
 950
 951         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 952 }
 953
 954 /**
 955  * Convert the @p action into a Verbs specification. This function assumes that
 956  * the input is valid and that there is space to insert the requested action
 957  * into the flow.
 958  *
 959  * @param[in] dev
 960  *   Pointer to the Ethernet device structure.
 961  * @param[in] action
 962  *   Action configuration.
 963  * @param[in] dev_flow
 964  *   Pointer to mlx5_flow.
 965  * @param[out] error
 966  *   Pointer to error structure.
 967  *
 968  * @return
 969  *   0 On success else a negative errno value is returned and rte_errno is set.
 970  */
 971 static int
 972 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
 973                                   const struct rte_flow_action *action,
 974                                   struct rte_eth_dev *dev,
 975                                   struct rte_flow_error *error)
 976 {
 977         const struct rte_flow_action_count *count = action->conf;
 978         struct rte_flow *flow = dev_flow->flow;
 979 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 980         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 981         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 982         struct ibv_flow_spec_counter_action counter = {
 983                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
 984                 .size = size,
 985         };
 986 #endif
 987
 988         if (!flow->counter) {
 989                 flow->counter = flow_verbs_counter_new(dev, count->shared,
 990                                                        count->id);
 991                 if (!flow->counter)
 992                         return rte_flow_error_set(error, rte_errno,
 993                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 994                                                   action,
 995                                                   "cannot get counter"
 996                                                   " context.");
 997         }
 998 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 999         counter.counter_set_handle = flow->counter->cs->handle;
1000         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1001 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1002         counter.counters = flow->counter->cs;
1003         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1004 #endif
1005         return 0;
1006 }
1007
1008 /**
1009  * Internal validation function. For validating both actions and items.
1010  *
1011  * @param[in] dev
1012  *   Pointer to the Ethernet device structure.
1013  * @param[in] attr
1014  *   Pointer to the flow attributes.
1015  * @param[in] items
1016  *   Pointer to the list of items.
1017  * @param[in] actions
1018  *   Pointer to the list of actions.
1019  * @param[in] external
1020  *   This flow rule is created by request external to PMD.
1021  * @param[out] error
1022  *   Pointer to the error structure.
1023  *
1024  * @return
1025  *   0 on success, a negative errno value otherwise and rte_errno is set.
1026  */
1027 static int
1028 flow_verbs_validate(struct rte_eth_dev *dev,
1029                     const struct rte_flow_attr *attr,
1030                     const struct rte_flow_item items[],
1031                     const struct rte_flow_action actions[],
1032                     bool external __rte_unused,
1033                     struct rte_flow_error *error)
1034 {
1035         int ret;
1036         uint64_t action_flags = 0;
1037         uint64_t item_flags = 0;
1038         uint64_t last_item = 0;
1039         uint8_t next_protocol = 0xff;
1040
1041         if (items == NULL)
1042                 return -1;
1043         ret = mlx5_flow_validate_attributes(dev, attr, error);
1044         if (ret < 0)
1045                 return ret;
1046         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1047                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1048                 int ret = 0;
1049
1050                 switch (items->type) {
1051                 case RTE_FLOW_ITEM_TYPE_VOID:
1052                         break;
1053                 case RTE_FLOW_ITEM_TYPE_ETH:
1054                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1055                                                           error);
1056                         if (ret < 0)
1057                                 return ret;
1058                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1059                                              MLX5_FLOW_LAYER_OUTER_L2;
1060                         break;
1061                 case RTE_FLOW_ITEM_TYPE_VLAN:
1062                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1063                                                            dev, error);
1064                         if (ret < 0)
1065                                 return ret;
1066                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1067                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1068                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1069                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1070                         break;
1071                 case RTE_FLOW_ITEM_TYPE_IPV4:
1072                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1073                                                            NULL, error);
1074                         if (ret < 0)
1075                                 return ret;
1076                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1077                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1078                         if (items->mask != NULL &&
1079                             ((const struct rte_flow_item_ipv4 *)
1080                              items->mask)->hdr.next_proto_id) {
1081                                 next_protocol =
1082                                         ((const struct rte_flow_item_ipv4 *)
1083                                          (items->spec))->hdr.next_proto_id;
1084                                 next_protocol &=
1085                                         ((const struct rte_flow_item_ipv4 *)
1086                                          (items->mask))->hdr.next_proto_id;
1087                         } else {
1088                                 /* Reset for inner layer. */
1089                                 next_protocol = 0xff;
1090                         }
1091                         break;
1092                 case RTE_FLOW_ITEM_TYPE_IPV6:
1093                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1094                                                            NULL, error);
1095                         if (ret < 0)
1096                                 return ret;
1097                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1098                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1099                         if (items->mask != NULL &&
1100                             ((const struct rte_flow_item_ipv6 *)
1101                              items->mask)->hdr.proto) {
1102                                 next_protocol =
1103                                         ((const struct rte_flow_item_ipv6 *)
1104                                          items->spec)->hdr.proto;
1105                                 next_protocol &=
1106                                         ((const struct rte_flow_item_ipv6 *)
1107                                          items->mask)->hdr.proto;
1108                         } else {
1109                                 /* Reset for inner layer. */
1110                                 next_protocol = 0xff;
1111                         }
1112                         break;
1113                 case RTE_FLOW_ITEM_TYPE_UDP:
1114                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1115                                                           next_protocol,
1116                                                           error);
1117                         if (ret < 0)
1118                                 return ret;
1119                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1120                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1121                         break;
1122                 case RTE_FLOW_ITEM_TYPE_TCP:
1123                         ret = mlx5_flow_validate_item_tcp
1124                                                 (items, item_flags,
1125                                                  next_protocol,
1126                                                  &rte_flow_item_tcp_mask,
1127                                                  error);
1128                         if (ret < 0)
1129                                 return ret;
1130                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1131                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1132                         break;
1133                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1134                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1135                                                             error);
1136                         if (ret < 0)
1137                                 return ret;
1138                         last_item = MLX5_FLOW_LAYER_VXLAN;
1139                         break;
1140                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1141                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1142                                                                 item_flags,
1143                                                                 dev, error);
1144                         if (ret < 0)
1145                                 return ret;
1146                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1147                         break;
1148                 case RTE_FLOW_ITEM_TYPE_GRE:
1149                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1150                                                           next_protocol, error);
1151                         if (ret < 0)
1152                                 return ret;
1153                         last_item = MLX5_FLOW_LAYER_GRE;
1154                         break;
1155                 case RTE_FLOW_ITEM_TYPE_MPLS:
1156                         ret = mlx5_flow_validate_item_mpls(dev, items,
1157                                                            item_flags,
1158                                                            last_item, error);
1159                         if (ret < 0)
1160                                 return ret;
1161                         last_item = MLX5_FLOW_LAYER_MPLS;
1162                         break;
1163                 default:
1164                         return rte_flow_error_set(error, ENOTSUP,
1165                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1166                                                   NULL, "item not supported");
1167                 }
1168                 item_flags |= last_item;
1169         }
1170         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1171                 switch (actions->type) {
1172                 case RTE_FLOW_ACTION_TYPE_VOID:
1173                         break;
1174                 case RTE_FLOW_ACTION_TYPE_FLAG:
1175                         ret = mlx5_flow_validate_action_flag(action_flags,
1176                                                              attr,
1177                                                              error);
1178                         if (ret < 0)
1179                                 return ret;
1180                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1181                         break;
1182                 case RTE_FLOW_ACTION_TYPE_MARK:
1183                         ret = mlx5_flow_validate_action_mark(actions,
1184                                                              action_flags,
1185                                                              attr,
1186                                                              error);
1187                         if (ret < 0)
1188                                 return ret;
1189                         action_flags |= MLX5_FLOW_ACTION_MARK;
1190                         break;
1191                 case RTE_FLOW_ACTION_TYPE_DROP:
1192                         ret = mlx5_flow_validate_action_drop(action_flags,
1193                                                              attr,
1194                                                              error);
1195                         if (ret < 0)
1196                                 return ret;
1197                         action_flags |= MLX5_FLOW_ACTION_DROP;
1198                         break;
1199                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1200                         ret = mlx5_flow_validate_action_queue(actions,
1201                                                               action_flags, dev,
1202                                                               attr,
1203                                                               error);
1204                         if (ret < 0)
1205                                 return ret;
1206                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1207                         break;
1208                 case RTE_FLOW_ACTION_TYPE_RSS:
1209                         ret = mlx5_flow_validate_action_rss(actions,
1210                                                             action_flags, dev,
1211                                                             attr, item_flags,
1212                                                             error);
1213                         if (ret < 0)
1214                                 return ret;
1215                         action_flags |= MLX5_FLOW_ACTION_RSS;
1216                         break;
1217                 case RTE_FLOW_ACTION_TYPE_COUNT:
1218                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1219                         if (ret < 0)
1220                                 return ret;
1221                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1222                         break;
1223                 default:
1224                         return rte_flow_error_set(error, ENOTSUP,
1225                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1226                                                   actions,
1227                                                   "action not supported");
1228                 }
1229         }
1230         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1231                 return rte_flow_error_set(error, EINVAL,
1232                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1233                                           "no fate action is found");
1234         return 0;
1235 }
1236
1237 /**
1238  * Calculate the required bytes that are needed for the action part of the verbs
1239  * flow.
1240  *
1241  * @param[in] actions
1242  *   Pointer to the list of actions.
1243  *
1244  * @return
1245  *   The size of the memory needed for all actions.
1246  */
1247 static int
1248 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1249 {
1250         int size = 0;
1251
1252         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1253                 switch (actions->type) {
1254                 case RTE_FLOW_ACTION_TYPE_VOID:
1255                         break;
1256                 case RTE_FLOW_ACTION_TYPE_FLAG:
1257                         size += sizeof(struct ibv_flow_spec_action_tag);
1258                         break;
1259                 case RTE_FLOW_ACTION_TYPE_MARK:
1260                         size += sizeof(struct ibv_flow_spec_action_tag);
1261                         break;
1262                 case RTE_FLOW_ACTION_TYPE_DROP:
1263                         size += sizeof(struct ibv_flow_spec_action_drop);
1264                         break;
1265                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1266                         break;
1267                 case RTE_FLOW_ACTION_TYPE_RSS:
1268                         break;
1269                 case RTE_FLOW_ACTION_TYPE_COUNT:
1270 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1271         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1272                         size += sizeof(struct ibv_flow_spec_counter_action);
1273 #endif
1274                         break;
1275                 default:
1276                         break;
1277                 }
1278         }
1279         return size;
1280 }
1281
1282 /**
1283  * Calculate the required bytes that are needed for the item part of the verbs
1284  * flow.
1285  *
1286  * @param[in] items
1287  *   Pointer to the list of items.
1288  *
1289  * @return
1290  *   The size of the memory needed for all items.
1291  */
1292 static int
1293 flow_verbs_get_items_size(const struct rte_flow_item items[])
1294 {
1295         int size = 0;
1296
1297         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1298                 switch (items->type) {
1299                 case RTE_FLOW_ITEM_TYPE_VOID:
1300                         break;
1301                 case RTE_FLOW_ITEM_TYPE_ETH:
1302                         size += sizeof(struct ibv_flow_spec_eth);
1303                         break;
1304                 case RTE_FLOW_ITEM_TYPE_VLAN:
1305                         size += sizeof(struct ibv_flow_spec_eth);
1306                         break;
1307                 case RTE_FLOW_ITEM_TYPE_IPV4:
1308                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1309                         break;
1310                 case RTE_FLOW_ITEM_TYPE_IPV6:
1311                         size += sizeof(struct ibv_flow_spec_ipv6);
1312                         break;
1313                 case RTE_FLOW_ITEM_TYPE_UDP:
1314                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1315                         break;
1316                 case RTE_FLOW_ITEM_TYPE_TCP:
1317                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1318                         break;
1319                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1320                         size += sizeof(struct ibv_flow_spec_tunnel);
1321                         break;
1322                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1323                         size += sizeof(struct ibv_flow_spec_tunnel);
1324                         break;
1325 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1326                 case RTE_FLOW_ITEM_TYPE_GRE:
1327                         size += sizeof(struct ibv_flow_spec_gre);
1328                         break;
1329                 case RTE_FLOW_ITEM_TYPE_MPLS:
1330                         size += sizeof(struct ibv_flow_spec_mpls);
1331                         break;
1332 #else
1333                 case RTE_FLOW_ITEM_TYPE_GRE:
1334                         size += sizeof(struct ibv_flow_spec_tunnel);
1335                         break;
1336 #endif
1337                 default:
1338                         break;
1339                 }
1340         }
1341         return size;
1342 }
1343
1344 /**
1345  * Internal preparation function. Allocate mlx5_flow with the required size.
1346  * The required size is calculate based on the actions and items. This function
1347  * also returns the detected actions and items for later use.
1348  *
1349  * @param[in] attr
1350  *   Pointer to the flow attributes.
1351  * @param[in] items
1352  *   Pointer to the list of items.
1353  * @param[in] actions
1354  *   Pointer to the list of actions.
1355  * @param[out] error
1356  *   Pointer to the error structure.
1357  *
1358  * @return
1359  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1360  *   is set.
1361  */
1362 static struct mlx5_flow *
1363 flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
1364                    const struct rte_flow_item items[],
1365                    const struct rte_flow_action actions[],
1366                    struct rte_flow_error *error)
1367 {
1368         uint32_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
1369         struct mlx5_flow *flow;
1370
1371         size += flow_verbs_get_actions_size(actions);
1372         size += flow_verbs_get_items_size(items);
1373         flow = rte_calloc(__func__, 1, size, 0);
1374         if (!flow) {
1375                 rte_flow_error_set(error, ENOMEM,
1376                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1377                                    "not enough memory to create flow");
1378                 return NULL;
1379         }
1380         flow->verbs.attr = (void *)(flow + 1);
1381         flow->verbs.specs =
1382                 (uint8_t *)(flow + 1) + sizeof(struct ibv_flow_attr);
1383         return flow;
1384 }
1385
1386 /**
1387  * Fill the flow with verb spec.
1388  *
1389  * @param[in] dev
1390  *   Pointer to Ethernet device.
1391  * @param[in, out] dev_flow
1392  *   Pointer to the mlx5 flow.
1393  * @param[in] attr
1394  *   Pointer to the flow attributes.
1395  * @param[in] items
1396  *   Pointer to the list of items.
1397  * @param[in] actions
1398  *   Pointer to the list of actions.
1399  * @param[out] error
1400  *   Pointer to the error structure.
1401  *
1402  * @return
1403  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1404  */
1405 static int
1406 flow_verbs_translate(struct rte_eth_dev *dev,
1407                      struct mlx5_flow *dev_flow,
1408                      const struct rte_flow_attr *attr,
1409                      const struct rte_flow_item items[],
1410                      const struct rte_flow_action actions[],
1411                      struct rte_flow_error *error)
1412 {
1413         struct rte_flow *flow = dev_flow->flow;
1414         uint64_t item_flags = 0;
1415         uint64_t action_flags = 0;
1416         uint64_t priority = attr->priority;
1417         uint32_t subpriority = 0;
1418         struct mlx5_priv *priv = dev->data->dev_private;
1419
1420         if (priority == MLX5_FLOW_PRIO_RSVD)
1421                 priority = priv->config.flow_prio - 1;
1422         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1423                 int ret;
1424
1425                 switch (actions->type) {
1426                 case RTE_FLOW_ACTION_TYPE_VOID:
1427                         break;
1428                 case RTE_FLOW_ACTION_TYPE_FLAG:
1429                         flow_verbs_translate_action_flag(dev_flow, actions);
1430                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1431                         break;
1432                 case RTE_FLOW_ACTION_TYPE_MARK:
1433                         flow_verbs_translate_action_mark(dev_flow, actions);
1434                         action_flags |= MLX5_FLOW_ACTION_MARK;
1435                         break;
1436                 case RTE_FLOW_ACTION_TYPE_DROP:
1437                         flow_verbs_translate_action_drop(dev_flow, actions);
1438                         action_flags |= MLX5_FLOW_ACTION_DROP;
1439                         break;
1440                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1441                         flow_verbs_translate_action_queue(dev_flow, actions);
1442                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1443                         break;
1444                 case RTE_FLOW_ACTION_TYPE_RSS:
1445                         flow_verbs_translate_action_rss(dev_flow, actions);
1446                         action_flags |= MLX5_FLOW_ACTION_RSS;
1447                         break;
1448                 case RTE_FLOW_ACTION_TYPE_COUNT:
1449                         ret = flow_verbs_translate_action_count(dev_flow,
1450                                                                 actions,
1451                                                                 dev, error);
1452                         if (ret < 0)
1453                                 return ret;
1454                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1455                         break;
1456                 default:
1457                         return rte_flow_error_set(error, ENOTSUP,
1458                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1459                                                   actions,
1460                                                   "action not supported");
1461                 }
1462         }
1463         flow->actions = action_flags;
1464         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1465                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1466
1467                 switch (items->type) {
1468                 case RTE_FLOW_ITEM_TYPE_VOID:
1469                         break;
1470                 case RTE_FLOW_ITEM_TYPE_ETH:
1471                         flow_verbs_translate_item_eth(dev_flow, items,
1472                                                       item_flags);
1473                         subpriority = MLX5_PRIORITY_MAP_L2;
1474                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1475                                                MLX5_FLOW_LAYER_OUTER_L2;
1476                         break;
1477                 case RTE_FLOW_ITEM_TYPE_VLAN:
1478                         flow_verbs_translate_item_vlan(dev_flow, items,
1479                                                        item_flags);
1480                         subpriority = MLX5_PRIORITY_MAP_L2;
1481                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1482                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1483                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1484                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1485                         break;
1486                 case RTE_FLOW_ITEM_TYPE_IPV4:
1487                         flow_verbs_translate_item_ipv4(dev_flow, items,
1488                                                        item_flags);
1489                         subpriority = MLX5_PRIORITY_MAP_L3;
1490                         dev_flow->verbs.hash_fields |=
1491                                 mlx5_flow_hashfields_adjust
1492                                         (dev_flow, tunnel,
1493                                          MLX5_IPV4_LAYER_TYPES,
1494                                          MLX5_IPV4_IBV_RX_HASH);
1495                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1496                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1497                         break;
1498                 case RTE_FLOW_ITEM_TYPE_IPV6:
1499                         flow_verbs_translate_item_ipv6(dev_flow, items,
1500                                                        item_flags);
1501                         subpriority = MLX5_PRIORITY_MAP_L3;
1502                         dev_flow->verbs.hash_fields |=
1503                                 mlx5_flow_hashfields_adjust
1504                                         (dev_flow, tunnel,
1505                                          MLX5_IPV6_LAYER_TYPES,
1506                                          MLX5_IPV6_IBV_RX_HASH);
1507                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1508                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1509                         break;
1510                 case RTE_FLOW_ITEM_TYPE_TCP:
1511                         flow_verbs_translate_item_tcp(dev_flow, items,
1512                                                       item_flags);
1513                         subpriority = MLX5_PRIORITY_MAP_L4;
1514                         dev_flow->verbs.hash_fields |=
1515                                 mlx5_flow_hashfields_adjust
1516                                         (dev_flow, tunnel, ETH_RSS_TCP,
1517                                          (IBV_RX_HASH_SRC_PORT_TCP |
1518                                           IBV_RX_HASH_DST_PORT_TCP));
1519                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1520                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1521                         break;
1522                 case RTE_FLOW_ITEM_TYPE_UDP:
1523                         flow_verbs_translate_item_udp(dev_flow, items,
1524                                                       item_flags);
1525                         subpriority = MLX5_PRIORITY_MAP_L4;
1526                         dev_flow->verbs.hash_fields |=
1527                                 mlx5_flow_hashfields_adjust
1528                                         (dev_flow, tunnel, ETH_RSS_UDP,
1529                                          (IBV_RX_HASH_SRC_PORT_UDP |
1530                                           IBV_RX_HASH_DST_PORT_UDP));
1531                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1532                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1533                         break;
1534                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1535                         flow_verbs_translate_item_vxlan(dev_flow, items,
1536                                                         item_flags);
1537                         subpriority = MLX5_PRIORITY_MAP_L2;
1538                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1539                         break;
1540                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1541                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1542                                                             item_flags);
1543                         subpriority = MLX5_PRIORITY_MAP_L2;
1544                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1545                         break;
1546                 case RTE_FLOW_ITEM_TYPE_GRE:
1547                         flow_verbs_translate_item_gre(dev_flow, items,
1548                                                       item_flags);
1549                         subpriority = MLX5_PRIORITY_MAP_L2;
1550                         item_flags |= MLX5_FLOW_LAYER_GRE;
1551                         break;
1552                 case RTE_FLOW_ITEM_TYPE_MPLS:
1553                         flow_verbs_translate_item_mpls(dev_flow, items,
1554                                                        item_flags);
1555                         subpriority = MLX5_PRIORITY_MAP_L2;
1556                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1557                         break;
1558                 default:
1559                         return rte_flow_error_set(error, ENOTSUP,
1560                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1561                                                   NULL,
1562                                                   "item not supported");
1563                 }
1564         }
1565         dev_flow->layers = item_flags;
1566         dev_flow->verbs.attr->priority =
1567                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1568         dev_flow->verbs.attr->port = (uint8_t)priv->ibv_port;
1569         return 0;
1570 }
1571
1572 /**
1573  * Remove the flow from the NIC but keeps it in memory.
1574  *
1575  * @param[in] dev
1576  *   Pointer to the Ethernet device structure.
1577  * @param[in, out] flow
1578  *   Pointer to flow structure.
1579  */
1580 static void
1581 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1582 {
1583         struct mlx5_flow_verbs *verbs;
1584         struct mlx5_flow *dev_flow;
1585
1586         if (!flow)
1587                 return;
1588         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1589                 verbs = &dev_flow->verbs;
1590                 if (verbs->flow) {
1591                         claim_zero(mlx5_glue->destroy_flow(verbs->flow));
1592                         verbs->flow = NULL;
1593                 }
1594                 if (verbs->hrxq) {
1595                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1596                                 mlx5_hrxq_drop_release(dev);
1597                         else
1598                                 mlx5_hrxq_release(dev, verbs->hrxq);
1599                         verbs->hrxq = NULL;
1600                 }
1601                 if (dev_flow->verbs.vf_vlan.tag &&
1602                     dev_flow->verbs.vf_vlan.created) {
1603                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1604                 }
1605         }
1606 }
1607
1608 /**
1609  * Remove the flow from the NIC and the memory.
1610  *
1611  * @param[in] dev
1612  *   Pointer to the Ethernet device structure.
1613  * @param[in, out] flow
1614  *   Pointer to flow structure.
1615  */
1616 static void
1617 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1618 {
1619         struct mlx5_flow *dev_flow;
1620
1621         if (!flow)
1622                 return;
1623         flow_verbs_remove(dev, flow);
1624         while (!LIST_EMPTY(&flow->dev_flows)) {
1625                 dev_flow = LIST_FIRST(&flow->dev_flows);
1626                 LIST_REMOVE(dev_flow, next);
1627                 rte_free(dev_flow);
1628         }
1629         if (flow->counter) {
1630                 flow_verbs_counter_release(dev, flow->counter);
1631                 flow->counter = NULL;
1632         }
1633 }
1634
1635 /**
1636  * Apply the flow to the NIC.
1637  *
1638  * @param[in] dev
1639  *   Pointer to the Ethernet device structure.
1640  * @param[in, out] flow
1641  *   Pointer to flow structure.
1642  * @param[out] error
1643  *   Pointer to error structure.
1644  *
1645  * @return
1646  *   0 on success, a negative errno value otherwise and rte_errno is set.
1647  */
1648 static int
1649 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1650                  struct rte_flow_error *error)
1651 {
1652         struct mlx5_priv *priv = dev->data->dev_private;
1653         struct mlx5_flow_verbs *verbs;
1654         struct mlx5_flow *dev_flow;
1655         int err;
1656
1657         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1658                 verbs = &dev_flow->verbs;
1659                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1660                         verbs->hrxq = mlx5_hrxq_drop_new(dev);
1661                         if (!verbs->hrxq) {
1662                                 rte_flow_error_set
1663                                         (error, errno,
1664                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1665                                          "cannot get drop hash queue");
1666                                 goto error;
1667                         }
1668                 } else {
1669                         struct mlx5_hrxq *hrxq;
1670
1671                         hrxq = mlx5_hrxq_get(dev, flow->key,
1672                                              MLX5_RSS_HASH_KEY_LEN,
1673                                              verbs->hash_fields,
1674                                              (*flow->queue),
1675                                              flow->rss.queue_num);
1676                         if (!hrxq)
1677                                 hrxq = mlx5_hrxq_new(dev, flow->key,
1678                                                      MLX5_RSS_HASH_KEY_LEN,
1679                                                      verbs->hash_fields,
1680                                                      (*flow->queue),
1681                                                      flow->rss.queue_num,
1682                                                      !!(dev_flow->layers &
1683                                                        MLX5_FLOW_LAYER_TUNNEL));
1684                         if (!hrxq) {
1685                                 rte_flow_error_set
1686                                         (error, rte_errno,
1687                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1688                                          "cannot get hash queue");
1689                                 goto error;
1690                         }
1691                         verbs->hrxq = hrxq;
1692                 }
1693                 verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp,
1694                                                      verbs->attr);
1695                 if (!verbs->flow) {
1696                         rte_flow_error_set(error, errno,
1697                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1698                                            NULL,
1699                                            "hardware refuses to create flow");
1700                         goto error;
1701                 }
1702                 if (priv->vmwa_context &&
1703                     dev_flow->verbs.vf_vlan.tag &&
1704                     !dev_flow->verbs.vf_vlan.created) {
1705                         /*
1706                          * The rule contains the VLAN pattern.
1707                          * For VF we are going to create VLAN
1708                          * interface to make hypervisor set correct
1709                          * e-Switch vport context.
1710                          */
1711                         mlx5_vlan_vmwa_acquire(dev, &dev_flow->verbs.vf_vlan);
1712                 }
1713         }
1714         return 0;
1715 error:
1716         err = rte_errno; /* Save rte_errno before cleanup. */
1717         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1718                 verbs = &dev_flow->verbs;
1719                 if (verbs->hrxq) {
1720                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1721                                 mlx5_hrxq_drop_release(dev);
1722                         else
1723                                 mlx5_hrxq_release(dev, verbs->hrxq);
1724                         verbs->hrxq = NULL;
1725                 }
1726                 if (dev_flow->verbs.vf_vlan.tag &&
1727                     dev_flow->verbs.vf_vlan.created) {
1728                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1729                 }
1730         }
1731         rte_errno = err; /* Restore rte_errno. */
1732         return -rte_errno;
1733 }
1734
1735 /**
1736  * Query a flow.
1737  *
1738  * @see rte_flow_query()
1739  * @see rte_flow_ops
1740  */
1741 static int
1742 flow_verbs_query(struct rte_eth_dev *dev,
1743                  struct rte_flow *flow,
1744                  const struct rte_flow_action *actions,
1745                  void *data,
1746                  struct rte_flow_error *error)
1747 {
1748         int ret = -EINVAL;
1749
1750         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1751                 switch (actions->type) {
1752                 case RTE_FLOW_ACTION_TYPE_VOID:
1753                         break;
1754                 case RTE_FLOW_ACTION_TYPE_COUNT:
1755                         ret = flow_verbs_counter_query(dev, flow, data, error);
1756                         break;
1757                 default:
1758                         return rte_flow_error_set(error, ENOTSUP,
1759                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1760                                                   actions,
1761                                                   "action not supported");
1762                 }
1763         }
1764         return ret;
1765 }
1766
1767 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1768         .validate = flow_verbs_validate,
1769         .prepare = flow_verbs_prepare,
1770         .translate = flow_verbs_translate,
1771         .apply = flow_verbs_apply,
1772         .remove = flow_verbs_remove,
1773         .destroy = flow_verbs_destroy,
1774         .query = flow_verbs_query,
1775 };