drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28
  29 #include "mlx5.h"
  30 #include "mlx5_defs.h"
  31 #include "mlx5_flow.h"
  32 #include "mlx5_glue.h"
  33 #include "mlx5_prm.h"
  34 #include "mlx5_rxtx.h"
  35
  36 #define VERBS_SPEC_INNER(item_flags) \
  37         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  38
  39 /**
  40  * Create Verbs flow counter with Verbs library.
  41  *
  42  * @param[in] dev
  43  *   Pointer to the Ethernet device structure.
  44  * @param[in, out] counter
  45  *   mlx5 flow counter object, contains the counter id,
  46  *   handle of created Verbs flow counter is returned
  47  *   in cs field (if counters are supported).
  48  *
  49  * @return
  50  *   0 On success else a negative errno value is returned
  51  *   and rte_errno is set.
  52  */
  53 static int
  54 flow_verbs_counter_create(struct rte_eth_dev *dev,
  55                           struct mlx5_flow_counter *counter)
  56 {
  57 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  58         struct mlx5_priv *priv = dev->data->dev_private;
  59         struct ibv_context *ctx = priv->sh->ctx;
  60         struct ibv_counter_set_init_attr init = {
  61                          .counter_set_id = counter->id};
  62
  63         counter->cs = mlx5_glue->create_counter_set(ctx, &init);
  64         if (!counter->cs) {
  65                 rte_errno = ENOTSUP;
  66                 return -ENOTSUP;
  67         }
  68         return 0;
  69 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
  70         struct mlx5_priv *priv = dev->data->dev_private;
  71         struct ibv_context *ctx = priv->sh->ctx;
  72         struct ibv_counters_init_attr init = {0};
  73         struct ibv_counter_attach_attr attach;
  74         int ret;
  75
  76         memset(&attach, 0, sizeof(attach));
  77         counter->cs = mlx5_glue->create_counters(ctx, &init);
  78         if (!counter->cs) {
  79                 rte_errno = ENOTSUP;
  80                 return -ENOTSUP;
  81         }
  82         attach.counter_desc = IBV_COUNTER_PACKETS;
  83         attach.index = 0;
  84         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
  85         if (!ret) {
  86                 attach.counter_desc = IBV_COUNTER_BYTES;
  87                 attach.index = 1;
  88                 ret = mlx5_glue->attach_counters
  89                                         (counter->cs, &attach, NULL);
  90         }
  91         if (ret) {
  92                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
  93                 counter->cs = NULL;
  94                 rte_errno = ret;
  95                 return -ret;
  96         }
  97         return 0;
  98 #else
  99         (void)dev;
 100         (void)counter;
 101         rte_errno = ENOTSUP;
 102         return -ENOTSUP;
 103 #endif
 104 }
 105
 106 /**
 107  * Get a flow counter.
 108  *
 109  * @param[in] dev
 110  *   Pointer to the Ethernet device structure.
 111  * @param[in] shared
 112  *   Indicate if this counter is shared with other flows.
 113  * @param[in] id
 114  *   Counter identifier.
 115  *
 116  * @return
 117  *   A pointer to the counter, NULL otherwise and rte_errno is set.
 118  */
 119 static struct mlx5_flow_counter *
 120 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 121 {
 122         struct mlx5_priv *priv = dev->data->dev_private;
 123         struct mlx5_flow_counter *cnt;
 124         int ret;
 125
 126         if (shared) {
 127                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
 128                         if (cnt->shared && cnt->id == id) {
 129                                 cnt->ref_cnt++;
 130                                 return cnt;
 131                         }
 132                 }
 133         }
 134         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 135         if (!cnt) {
 136                 rte_errno = ENOMEM;
 137                 return NULL;
 138         }
 139         cnt->id = id;
 140         cnt->shared = shared;
 141         cnt->ref_cnt = 1;
 142         cnt->hits = 0;
 143         cnt->bytes = 0;
 144         /* Create counter with Verbs. */
 145         ret = flow_verbs_counter_create(dev, cnt);
 146         if (!ret) {
 147                 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
 148                 return cnt;
 149         }
 150         /* Some error occurred in Verbs library. */
 151         rte_free(cnt);
 152         rte_errno = -ret;
 153         return NULL;
 154 }
 155
 156 /**
 157  * Release a flow counter.
 158  *
 159  * @param[in] dev
 160  *   Pointer to the Ethernet device structure.
 161  * @param[in] counter
 162  *   Pointer to the counter handler.
 163  */
 164 static void
 165 flow_verbs_counter_release(struct rte_eth_dev *dev,
 166                            struct mlx5_flow_counter *counter)
 167 {
 168         struct mlx5_priv *priv = dev->data->dev_private;
 169
 170         if (--counter->ref_cnt == 0) {
 171 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 172                 claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
 173 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 174                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 175 #endif
 176                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
 177                 rte_free(counter);
 178         }
 179 }
 180
 181 /**
 182  * Query a flow counter via Verbs library call.
 183  *
 184  * @see rte_flow_query()
 185  * @see rte_flow_ops
 186  */
 187 static int
 188 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 189                          struct rte_flow *flow, void *data,
 190                          struct rte_flow_error *error)
 191 {
 192 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 193         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 194         if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 195                 struct rte_flow_query_count *qc = data;
 196                 uint64_t counters[2] = {0, 0};
 197 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 198                 struct ibv_query_counter_set_attr query_cs_attr = {
 199                         .cs = flow->counter->cs,
 200                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 201                 };
 202                 struct ibv_counter_set_data query_out = {
 203                         .out = counters,
 204                         .outlen = 2 * sizeof(uint64_t),
 205                 };
 206                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 207                                                        &query_out);
 208 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 209                 int err = mlx5_glue->query_counters
 210                                (flow->counter->cs, counters,
 211                                 RTE_DIM(counters),
 212                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 213 #endif
 214                 if (err)
 215                         return rte_flow_error_set
 216                                 (error, err,
 217                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 218                                  NULL,
 219                                  "cannot read counter");
 220                 qc->hits_set = 1;
 221                 qc->bytes_set = 1;
 222                 qc->hits = counters[0] - flow->counter->hits;
 223                 qc->bytes = counters[1] - flow->counter->bytes;
 224                 if (qc->reset) {
 225                         flow->counter->hits = counters[0];
 226                         flow->counter->bytes = counters[1];
 227                 }
 228                 return 0;
 229         }
 230         return rte_flow_error_set(error, EINVAL,
 231                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 232                                   NULL,
 233                                   "flow does not have counter");
 234 #else
 235         (void)flow;
 236         (void)data;
 237         return rte_flow_error_set(error, ENOTSUP,
 238                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 239                                   NULL,
 240                                   "counters are not available");
 241 #endif
 242 }
 243
 244 /**
 245  * Add a verbs item specification into @p verbs.
 246  *
 247  * @param[out] verbs
 248  *   Pointer to verbs structure.
 249  * @param[in] src
 250  *   Create specification.
 251  * @param[in] size
 252  *   Size in bytes of the specification to copy.
 253  */
 254 static void
 255 flow_verbs_spec_add(struct mlx5_flow_verbs *verbs, void *src, unsigned int size)
 256 {
 257         void *dst;
 258
 259         if (!verbs)
 260                 return;
 261         assert(verbs->specs);
 262         dst = (void *)(verbs->specs + verbs->size);
 263         memcpy(dst, src, size);
 264         ++verbs->attr->num_of_specs;
 265         verbs->size += size;
 266 }
 267
 268 /**
 269  * Convert the @p item into a Verbs specification. This function assumes that
 270  * the input is valid and that there is space to insert the requested item
 271  * into the flow.
 272  *
 273  * @param[in, out] dev_flow
 274  *   Pointer to dev_flow structure.
 275  * @param[in] item
 276  *   Item specification.
 277  * @param[in] item_flags
 278  *   Parsed item flags.
 279  */
 280 static void
 281 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 282                               const struct rte_flow_item *item,
 283                               uint64_t item_flags)
 284 {
 285         const struct rte_flow_item_eth *spec = item->spec;
 286         const struct rte_flow_item_eth *mask = item->mask;
 287         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 288         struct ibv_flow_spec_eth eth = {
 289                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 290                 .size = size,
 291         };
 292
 293         if (!mask)
 294                 mask = &rte_flow_item_eth_mask;
 295         if (spec) {
 296                 unsigned int i;
 297
 298                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 299                         RTE_ETHER_ADDR_LEN);
 300                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 301                         RTE_ETHER_ADDR_LEN);
 302                 eth.val.ether_type = spec->type;
 303                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 304                         RTE_ETHER_ADDR_LEN);
 305                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 306                         RTE_ETHER_ADDR_LEN);
 307                 eth.mask.ether_type = mask->type;
 308                 /* Remove unwanted bits from values. */
 309                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 310                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 311                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 312                 }
 313                 eth.val.ether_type &= eth.mask.ether_type;
 314         }
 315         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 316 }
 317
 318 /**
 319  * Update the VLAN tag in the Verbs Ethernet specification.
 320  * This function assumes that the input is valid and there is space to add
 321  * the requested item.
 322  *
 323  * @param[in, out] attr
 324  *   Pointer to Verbs attributes structure.
 325  * @param[in] eth
 326  *   Verbs structure containing the VLAN information to copy.
 327  */
 328 static void
 329 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 330                             struct ibv_flow_spec_eth *eth)
 331 {
 332         unsigned int i;
 333         const enum ibv_flow_spec_type search = eth->type;
 334         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 335                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 336
 337         for (i = 0; i != attr->num_of_specs; ++i) {
 338                 if (hdr->type == search) {
 339                         struct ibv_flow_spec_eth *e =
 340                                 (struct ibv_flow_spec_eth *)hdr;
 341
 342                         e->val.vlan_tag = eth->val.vlan_tag;
 343                         e->mask.vlan_tag = eth->mask.vlan_tag;
 344                         e->val.ether_type = eth->val.ether_type;
 345                         e->mask.ether_type = eth->mask.ether_type;
 346                         break;
 347                 }
 348                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 349         }
 350 }
 351
 352 /**
 353  * Convert the @p item into a Verbs specification. This function assumes that
 354  * the input is valid and that there is space to insert the requested item
 355  * into the flow.
 356  *
 357  * @param[in, out] dev_flow
 358  *   Pointer to dev_flow structure.
 359  * @param[in] item
 360  *   Item specification.
 361  * @param[in] item_flags
 362  *   Parsed item flags.
 363  */
 364 static void
 365 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 366                                const struct rte_flow_item *item,
 367                                uint64_t item_flags)
 368 {
 369         const struct rte_flow_item_vlan *spec = item->spec;
 370         const struct rte_flow_item_vlan *mask = item->mask;
 371         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 372         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 373         struct ibv_flow_spec_eth eth = {
 374                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 375                 .size = size,
 376         };
 377         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 378                                       MLX5_FLOW_LAYER_OUTER_L2;
 379
 380         if (!mask)
 381                 mask = &rte_flow_item_vlan_mask;
 382         if (spec) {
 383                 eth.val.vlan_tag = spec->tci;
 384                 eth.mask.vlan_tag = mask->tci;
 385                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 386                 eth.val.ether_type = spec->inner_type;
 387                 eth.mask.ether_type = mask->inner_type;
 388                 eth.val.ether_type &= eth.mask.ether_type;
 389         }
 390         if (!(item_flags & l2m))
 391                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 392         else
 393                 flow_verbs_item_vlan_update(dev_flow->verbs.attr, &eth);
 394         if (!tunnel)
 395                 dev_flow->verbs.vf_vlan.tag =
 396                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 397 }
 398
 399 /**
 400  * Convert the @p item into a Verbs specification. This function assumes that
 401  * the input is valid and that there is space to insert the requested item
 402  * into the flow.
 403  *
 404  * @param[in, out] dev_flow
 405  *   Pointer to dev_flow structure.
 406  * @param[in] item
 407  *   Item specification.
 408  * @param[in] item_flags
 409  *   Parsed item flags.
 410  */
 411 static void
 412 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 413                                const struct rte_flow_item *item,
 414                                uint64_t item_flags)
 415 {
 416         const struct rte_flow_item_ipv4 *spec = item->spec;
 417         const struct rte_flow_item_ipv4 *mask = item->mask;
 418         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 419         struct ibv_flow_spec_ipv4_ext ipv4 = {
 420                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 421                 .size = size,
 422         };
 423
 424         if (!mask)
 425                 mask = &rte_flow_item_ipv4_mask;
 426         if (spec) {
 427                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 428                         .src_ip = spec->hdr.src_addr,
 429                         .dst_ip = spec->hdr.dst_addr,
 430                         .proto = spec->hdr.next_proto_id,
 431                         .tos = spec->hdr.type_of_service,
 432                 };
 433                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 434                         .src_ip = mask->hdr.src_addr,
 435                         .dst_ip = mask->hdr.dst_addr,
 436                         .proto = mask->hdr.next_proto_id,
 437                         .tos = mask->hdr.type_of_service,
 438                 };
 439                 /* Remove unwanted bits from values. */
 440                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 441                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 442                 ipv4.val.proto &= ipv4.mask.proto;
 443                 ipv4.val.tos &= ipv4.mask.tos;
 444         }
 445         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 446 }
 447
 448 /**
 449  * Convert the @p item into a Verbs specification. This function assumes that
 450  * the input is valid and that there is space to insert the requested item
 451  * into the flow.
 452  *
 453  * @param[in, out] dev_flow
 454  *   Pointer to dev_flow structure.
 455  * @param[in] item
 456  *   Item specification.
 457  * @param[in] item_flags
 458  *   Parsed item flags.
 459  */
 460 static void
 461 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 462                                const struct rte_flow_item *item,
 463                                uint64_t item_flags)
 464 {
 465         const struct rte_flow_item_ipv6 *spec = item->spec;
 466         const struct rte_flow_item_ipv6 *mask = item->mask;
 467         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 468         struct ibv_flow_spec_ipv6 ipv6 = {
 469                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 470                 .size = size,
 471         };
 472
 473         if (!mask)
 474                 mask = &rte_flow_item_ipv6_mask;
 475         if (spec) {
 476                 unsigned int i;
 477                 uint32_t vtc_flow_val;
 478                 uint32_t vtc_flow_mask;
 479
 480                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 481                        RTE_DIM(ipv6.val.src_ip));
 482                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 483                        RTE_DIM(ipv6.val.dst_ip));
 484                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 485                        RTE_DIM(ipv6.mask.src_ip));
 486                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 487                        RTE_DIM(ipv6.mask.dst_ip));
 488                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 489                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 490                 ipv6.val.flow_label =
 491                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 492                                          RTE_IPV6_HDR_FL_SHIFT);
 493                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 494                                          RTE_IPV6_HDR_TC_SHIFT;
 495                 ipv6.val.next_hdr = spec->hdr.proto;
 496                 ipv6.val.hop_limit = spec->hdr.hop_limits;
 497                 ipv6.mask.flow_label =
 498                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 499                                          RTE_IPV6_HDR_FL_SHIFT);
 500                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 501                                           RTE_IPV6_HDR_TC_SHIFT;
 502                 ipv6.mask.next_hdr = mask->hdr.proto;
 503                 ipv6.mask.hop_limit = mask->hdr.hop_limits;
 504                 /* Remove unwanted bits from values. */
 505                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 506                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 507                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 508                 }
 509                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 510                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 511                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 512                 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 513         }
 514         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 515 }
 516
 517 /**
 518  * Convert the @p item into a Verbs specification. This function assumes that
 519  * the input is valid and that there is space to insert the requested item
 520  * into the flow.
 521  *
 522  * @param[in, out] dev_flow
 523  *   Pointer to dev_flow structure.
 524  * @param[in] item
 525  *   Item specification.
 526  * @param[in] item_flags
 527  *   Parsed item flags.
 528  */
 529 static void
 530 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 531                               const struct rte_flow_item *item,
 532                               uint64_t item_flags __rte_unused)
 533 {
 534         const struct rte_flow_item_tcp *spec = item->spec;
 535         const struct rte_flow_item_tcp *mask = item->mask;
 536         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 537         struct ibv_flow_spec_tcp_udp tcp = {
 538                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 539                 .size = size,
 540         };
 541
 542         if (!mask)
 543                 mask = &rte_flow_item_tcp_mask;
 544         if (spec) {
 545                 tcp.val.dst_port = spec->hdr.dst_port;
 546                 tcp.val.src_port = spec->hdr.src_port;
 547                 tcp.mask.dst_port = mask->hdr.dst_port;
 548                 tcp.mask.src_port = mask->hdr.src_port;
 549                 /* Remove unwanted bits from values. */
 550                 tcp.val.src_port &= tcp.mask.src_port;
 551                 tcp.val.dst_port &= tcp.mask.dst_port;
 552         }
 553         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 554 }
 555
 556 /**
 557  * Convert the @p item into a Verbs specification. This function assumes that
 558  * the input is valid and that there is space to insert the requested item
 559  * into the flow.
 560  *
 561  * @param[in, out] dev_flow
 562  *   Pointer to dev_flow structure.
 563  * @param[in] item
 564  *   Item specification.
 565  * @param[in] item_flags
 566  *   Parsed item flags.
 567  */
 568 static void
 569 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 570                               const struct rte_flow_item *item,
 571                               uint64_t item_flags __rte_unused)
 572 {
 573         const struct rte_flow_item_udp *spec = item->spec;
 574         const struct rte_flow_item_udp *mask = item->mask;
 575         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 576         struct ibv_flow_spec_tcp_udp udp = {
 577                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 578                 .size = size,
 579         };
 580
 581         if (!mask)
 582                 mask = &rte_flow_item_udp_mask;
 583         if (spec) {
 584                 udp.val.dst_port = spec->hdr.dst_port;
 585                 udp.val.src_port = spec->hdr.src_port;
 586                 udp.mask.dst_port = mask->hdr.dst_port;
 587                 udp.mask.src_port = mask->hdr.src_port;
 588                 /* Remove unwanted bits from values. */
 589                 udp.val.src_port &= udp.mask.src_port;
 590                 udp.val.dst_port &= udp.mask.dst_port;
 591         }
 592         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 593 }
 594
 595 /**
 596  * Convert the @p item into a Verbs specification. This function assumes that
 597  * the input is valid and that there is space to insert the requested item
 598  * into the flow.
 599  *
 600  * @param[in, out] dev_flow
 601  *   Pointer to dev_flow structure.
 602  * @param[in] item
 603  *   Item specification.
 604  * @param[in] item_flags
 605  *   Parsed item flags.
 606  */
 607 static void
 608 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 609                                 const struct rte_flow_item *item,
 610                                 uint64_t item_flags __rte_unused)
 611 {
 612         const struct rte_flow_item_vxlan *spec = item->spec;
 613         const struct rte_flow_item_vxlan *mask = item->mask;
 614         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 615         struct ibv_flow_spec_tunnel vxlan = {
 616                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 617                 .size = size,
 618         };
 619         union vni {
 620                 uint32_t vlan_id;
 621                 uint8_t vni[4];
 622         } id = { .vlan_id = 0, };
 623
 624         if (!mask)
 625                 mask = &rte_flow_item_vxlan_mask;
 626         if (spec) {
 627                 memcpy(&id.vni[1], spec->vni, 3);
 628                 vxlan.val.tunnel_id = id.vlan_id;
 629                 memcpy(&id.vni[1], mask->vni, 3);
 630                 vxlan.mask.tunnel_id = id.vlan_id;
 631                 /* Remove unwanted bits from values. */
 632                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 633         }
 634         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 635 }
 636
 637 /**
 638  * Convert the @p item into a Verbs specification. This function assumes that
 639  * the input is valid and that there is space to insert the requested item
 640  * into the flow.
 641  *
 642  * @param[in, out] dev_flow
 643  *   Pointer to dev_flow structure.
 644  * @param[in] item
 645  *   Item specification.
 646  * @param[in] item_flags
 647  *   Parsed item flags.
 648  */
 649 static void
 650 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 651                                     const struct rte_flow_item *item,
 652                                     uint64_t item_flags __rte_unused)
 653 {
 654         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 655         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 656         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 657         struct ibv_flow_spec_tunnel vxlan_gpe = {
 658                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 659                 .size = size,
 660         };
 661         union vni {
 662                 uint32_t vlan_id;
 663                 uint8_t vni[4];
 664         } id = { .vlan_id = 0, };
 665
 666         if (!mask)
 667                 mask = &rte_flow_item_vxlan_gpe_mask;
 668         if (spec) {
 669                 memcpy(&id.vni[1], spec->vni, 3);
 670                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 671                 memcpy(&id.vni[1], mask->vni, 3);
 672                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 673                 /* Remove unwanted bits from values. */
 674                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 675         }
 676         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 677 }
 678
 679 /**
 680  * Update the protocol in Verbs IPv4/IPv6 spec.
 681  *
 682  * @param[in, out] attr
 683  *   Pointer to Verbs attributes structure.
 684  * @param[in] search
 685  *   Specification type to search in order to update the IP protocol.
 686  * @param[in] protocol
 687  *   Protocol value to set if none is present in the specification.
 688  */
 689 static void
 690 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 691                                        enum ibv_flow_spec_type search,
 692                                        uint8_t protocol)
 693 {
 694         unsigned int i;
 695         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 696                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 697
 698         if (!attr)
 699                 return;
 700         for (i = 0; i != attr->num_of_specs; ++i) {
 701                 if (hdr->type == search) {
 702                         union {
 703                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 704                                 struct ibv_flow_spec_ipv6 *ipv6;
 705                         } ip;
 706
 707                         switch (search) {
 708                         case IBV_FLOW_SPEC_IPV4_EXT:
 709                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 710                                 if (!ip.ipv4->val.proto) {
 711                                         ip.ipv4->val.proto = protocol;
 712                                         ip.ipv4->mask.proto = 0xff;
 713                                 }
 714                                 break;
 715                         case IBV_FLOW_SPEC_IPV6:
 716                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 717                                 if (!ip.ipv6->val.next_hdr) {
 718                                         ip.ipv6->val.next_hdr = protocol;
 719                                         ip.ipv6->mask.next_hdr = 0xff;
 720                                 }
 721                                 break;
 722                         default:
 723                                 break;
 724                         }
 725                         break;
 726                 }
 727                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 728         }
 729 }
 730
 731 /**
 732  * Convert the @p item into a Verbs specification. This function assumes that
 733  * the input is valid and that there is space to insert the requested item
 734  * into the flow.
 735  *
 736  * @param[in, out] dev_flow
 737  *   Pointer to dev_flow structure.
 738  * @param[in] item
 739  *   Item specification.
 740  * @param[in] item_flags
 741  *   Parsed item flags.
 742  */
 743 static void
 744 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 745                               const struct rte_flow_item *item __rte_unused,
 746                               uint64_t item_flags)
 747 {
 748         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
 749 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 750         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 751         struct ibv_flow_spec_tunnel tunnel = {
 752                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 753                 .size = size,
 754         };
 755 #else
 756         const struct rte_flow_item_gre *spec = item->spec;
 757         const struct rte_flow_item_gre *mask = item->mask;
 758         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 759         struct ibv_flow_spec_gre tunnel = {
 760                 .type = IBV_FLOW_SPEC_GRE,
 761                 .size = size,
 762         };
 763
 764         if (!mask)
 765                 mask = &rte_flow_item_gre_mask;
 766         if (spec) {
 767                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 768                 tunnel.val.protocol = spec->protocol;
 769                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 770                 tunnel.mask.protocol = mask->protocol;
 771                 /* Remove unwanted bits from values. */
 772                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 773                 tunnel.val.protocol &= tunnel.mask.protocol;
 774                 tunnel.val.key &= tunnel.mask.key;
 775         }
 776 #endif
 777         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 778                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 779                                                        IBV_FLOW_SPEC_IPV4_EXT,
 780                                                        IPPROTO_GRE);
 781         else
 782                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 783                                                        IBV_FLOW_SPEC_IPV6,
 784                                                        IPPROTO_GRE);
 785         flow_verbs_spec_add(verbs, &tunnel, size);
 786 }
 787
 788 /**
 789  * Convert the @p action into a Verbs specification. This function assumes that
 790  * the input is valid and that there is space to insert the requested action
 791  * into the flow. This function also return the action that was added.
 792  *
 793  * @param[in, out] dev_flow
 794  *   Pointer to dev_flow structure.
 795  * @param[in] item
 796  *   Item specification.
 797  * @param[in] item_flags
 798  *   Parsed item flags.
 799  */
 800 static void
 801 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
 802                                const struct rte_flow_item *item __rte_unused,
 803                                uint64_t item_flags __rte_unused)
 804 {
 805 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 806         const struct rte_flow_item_mpls *spec = item->spec;
 807         const struct rte_flow_item_mpls *mask = item->mask;
 808         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 809         struct ibv_flow_spec_mpls mpls = {
 810                 .type = IBV_FLOW_SPEC_MPLS,
 811                 .size = size,
 812         };
 813
 814         if (!mask)
 815                 mask = &rte_flow_item_mpls_mask;
 816         if (spec) {
 817                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 818                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 819                 /* Remove unwanted bits from values.  */
 820                 mpls.val.label &= mpls.mask.label;
 821         }
 822         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
 823 #endif
 824 }
 825
 826 /**
 827  * Convert the @p action into a Verbs specification. This function assumes that
 828  * the input is valid and that there is space to insert the requested action
 829  * into the flow.
 830  *
 831  * @param[in] dev_flow
 832  *   Pointer to mlx5_flow.
 833  * @param[in] action
 834  *   Action configuration.
 835  */
 836 static void
 837 flow_verbs_translate_action_drop
 838         (struct mlx5_flow *dev_flow,
 839          const struct rte_flow_action *action __rte_unused)
 840 {
 841         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 842         struct ibv_flow_spec_action_drop drop = {
 843                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 844                         .size = size,
 845         };
 846
 847         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
 848 }
 849
 850 /**
 851  * Convert the @p action into a Verbs specification. This function assumes that
 852  * the input is valid and that there is space to insert the requested action
 853  * into the flow.
 854  *
 855  * @param[in] dev_flow
 856  *   Pointer to mlx5_flow.
 857  * @param[in] action
 858  *   Action configuration.
 859  */
 860 static void
 861 flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow,
 862                                   const struct rte_flow_action *action)
 863 {
 864         const struct rte_flow_action_queue *queue = action->conf;
 865         struct rte_flow *flow = dev_flow->flow;
 866
 867         if (flow->queue)
 868                 (*flow->queue)[0] = queue->index;
 869         flow->rss.queue_num = 1;
 870 }
 871
 872 /**
 873  * Convert the @p action into a Verbs specification. This function assumes that
 874  * the input is valid and that there is space to insert the requested action
 875  * into the flow.
 876  *
 877  * @param[in] action
 878  *   Action configuration.
 879  * @param[in, out] action_flags
 880  *   Pointer to the detected actions.
 881  * @param[in] dev_flow
 882  *   Pointer to mlx5_flow.
 883  */
 884 static void
 885 flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow,
 886                                 const struct rte_flow_action *action)
 887 {
 888         const struct rte_flow_action_rss *rss = action->conf;
 889         const uint8_t *rss_key;
 890         struct rte_flow *flow = dev_flow->flow;
 891
 892         if (flow->queue)
 893                 memcpy((*flow->queue), rss->queue,
 894                        rss->queue_num * sizeof(uint16_t));
 895         flow->rss.queue_num = rss->queue_num;
 896         /* NULL RSS key indicates default RSS key. */
 897         rss_key = !rss->key ? rss_hash_default_key : rss->key;
 898         memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
 899         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
 900         flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
 901         flow->rss.level = rss->level;
 902 }
 903
 904 /**
 905  * Convert the @p action into a Verbs specification. This function assumes that
 906  * the input is valid and that there is space to insert the requested action
 907  * into the flow.
 908  *
 909  * @param[in] dev_flow
 910  *   Pointer to mlx5_flow.
 911  * @param[in] action
 912  *   Action configuration.
 913  */
 914 static void
 915 flow_verbs_translate_action_flag
 916         (struct mlx5_flow *dev_flow,
 917          const struct rte_flow_action *action __rte_unused)
 918 {
 919         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 920         struct ibv_flow_spec_action_tag tag = {
 921                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 922                 .size = size,
 923                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 924         };
 925
 926         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 927 }
 928
 929 /**
 930  * Convert the @p action into a Verbs specification. This function assumes that
 931  * the input is valid and that there is space to insert the requested action
 932  * into the flow.
 933  *
 934  * @param[in] dev_flow
 935  *   Pointer to mlx5_flow.
 936  * @param[in] action
 937  *   Action configuration.
 938  */
 939 static void
 940 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
 941                                  const struct rte_flow_action *action)
 942 {
 943         const struct rte_flow_action_mark *mark = action->conf;
 944         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 945         struct ibv_flow_spec_action_tag tag = {
 946                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 947                 .size = size,
 948                 .tag_id = mlx5_flow_mark_set(mark->id),
 949         };
 950
 951         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 952 }
 953
 954 /**
 955  * Convert the @p action into a Verbs specification. This function assumes that
 956  * the input is valid and that there is space to insert the requested action
 957  * into the flow.
 958  *
 959  * @param[in] dev
 960  *   Pointer to the Ethernet device structure.
 961  * @param[in] action
 962  *   Action configuration.
 963  * @param[in] dev_flow
 964  *   Pointer to mlx5_flow.
 965  * @param[out] error
 966  *   Pointer to error structure.
 967  *
 968  * @return
 969  *   0 On success else a negative errno value is returned and rte_errno is set.
 970  */
 971 static int
 972 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
 973                                   const struct rte_flow_action *action,
 974                                   struct rte_eth_dev *dev,
 975                                   struct rte_flow_error *error)
 976 {
 977         const struct rte_flow_action_count *count = action->conf;
 978         struct rte_flow *flow = dev_flow->flow;
 979 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 980         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 981         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 982         struct ibv_flow_spec_counter_action counter = {
 983                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
 984                 .size = size,
 985         };
 986 #endif
 987
 988         if (!flow->counter) {
 989                 flow->counter = flow_verbs_counter_new(dev, count->shared,
 990                                                        count->id);
 991                 if (!flow->counter)
 992                         return rte_flow_error_set(error, rte_errno,
 993                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 994                                                   action,
 995                                                   "cannot get counter"
 996                                                   " context.");
 997         }
 998 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 999         counter.counter_set_handle = flow->counter->cs->handle;
1000         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1001 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1002         counter.counters = flow->counter->cs;
1003         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1004 #endif
1005         return 0;
1006 }
1007
1008 /**
1009  * Internal validation function. For validating both actions and items.
1010  *
1011  * @param[in] dev
1012  *   Pointer to the Ethernet device structure.
1013  * @param[in] attr
1014  *   Pointer to the flow attributes.
1015  * @param[in] items
1016  *   Pointer to the list of items.
1017  * @param[in] actions
1018  *   Pointer to the list of actions.
1019  * @param[out] error
1020  *   Pointer to the error structure.
1021  *
1022  * @return
1023  *   0 on success, a negative errno value otherwise and rte_errno is set.
1024  */
1025 static int
1026 flow_verbs_validate(struct rte_eth_dev *dev,
1027                     const struct rte_flow_attr *attr,
1028                     const struct rte_flow_item items[],
1029                     const struct rte_flow_action actions[],
1030                     struct rte_flow_error *error)
1031 {
1032         int ret;
1033         uint64_t action_flags = 0;
1034         uint64_t item_flags = 0;
1035         uint64_t last_item = 0;
1036         uint8_t next_protocol = 0xff;
1037
1038         if (items == NULL)
1039                 return -1;
1040         ret = mlx5_flow_validate_attributes(dev, attr, error);
1041         if (ret < 0)
1042                 return ret;
1043         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1044                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1045                 int ret = 0;
1046
1047                 switch (items->type) {
1048                 case RTE_FLOW_ITEM_TYPE_VOID:
1049                         break;
1050                 case RTE_FLOW_ITEM_TYPE_ETH:
1051                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1052                                                           error);
1053                         if (ret < 0)
1054                                 return ret;
1055                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1056                                              MLX5_FLOW_LAYER_OUTER_L2;
1057                         break;
1058                 case RTE_FLOW_ITEM_TYPE_VLAN:
1059                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1060                                                            dev, error);
1061                         if (ret < 0)
1062                                 return ret;
1063                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1064                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1065                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1066                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1067                         break;
1068                 case RTE_FLOW_ITEM_TYPE_IPV4:
1069                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1070                                                            NULL, error);
1071                         if (ret < 0)
1072                                 return ret;
1073                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1074                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1075                         if (items->mask != NULL &&
1076                             ((const struct rte_flow_item_ipv4 *)
1077                              items->mask)->hdr.next_proto_id) {
1078                                 next_protocol =
1079                                         ((const struct rte_flow_item_ipv4 *)
1080                                          (items->spec))->hdr.next_proto_id;
1081                                 next_protocol &=
1082                                         ((const struct rte_flow_item_ipv4 *)
1083                                          (items->mask))->hdr.next_proto_id;
1084                         } else {
1085                                 /* Reset for inner layer. */
1086                                 next_protocol = 0xff;
1087                         }
1088                         break;
1089                 case RTE_FLOW_ITEM_TYPE_IPV6:
1090                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1091                                                            NULL, error);
1092                         if (ret < 0)
1093                                 return ret;
1094                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1095                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1096                         if (items->mask != NULL &&
1097                             ((const struct rte_flow_item_ipv6 *)
1098                              items->mask)->hdr.proto) {
1099                                 next_protocol =
1100                                         ((const struct rte_flow_item_ipv6 *)
1101                                          items->spec)->hdr.proto;
1102                                 next_protocol &=
1103                                         ((const struct rte_flow_item_ipv6 *)
1104                                          items->mask)->hdr.proto;
1105                         } else {
1106                                 /* Reset for inner layer. */
1107                                 next_protocol = 0xff;
1108                         }
1109                         break;
1110                 case RTE_FLOW_ITEM_TYPE_UDP:
1111                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1112                                                           next_protocol,
1113                                                           error);
1114                         if (ret < 0)
1115                                 return ret;
1116                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1117                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1118                         break;
1119                 case RTE_FLOW_ITEM_TYPE_TCP:
1120                         ret = mlx5_flow_validate_item_tcp
1121                                                 (items, item_flags,
1122                                                  next_protocol,
1123                                                  &rte_flow_item_tcp_mask,
1124                                                  error);
1125                         if (ret < 0)
1126                                 return ret;
1127                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1128                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1129                         break;
1130                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1131                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1132                                                             error);
1133                         if (ret < 0)
1134                                 return ret;
1135                         last_item = MLX5_FLOW_LAYER_VXLAN;
1136                         break;
1137                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1138                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1139                                                                 item_flags,
1140                                                                 dev, error);
1141                         if (ret < 0)
1142                                 return ret;
1143                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1144                         break;
1145                 case RTE_FLOW_ITEM_TYPE_GRE:
1146                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1147                                                           next_protocol, error);
1148                         if (ret < 0)
1149                                 return ret;
1150                         last_item = MLX5_FLOW_LAYER_GRE;
1151                         break;
1152                 case RTE_FLOW_ITEM_TYPE_MPLS:
1153                         ret = mlx5_flow_validate_item_mpls(dev, items,
1154                                                            item_flags,
1155                                                            last_item, error);
1156                         if (ret < 0)
1157                                 return ret;
1158                         last_item = MLX5_FLOW_LAYER_MPLS;
1159                         break;
1160                 default:
1161                         return rte_flow_error_set(error, ENOTSUP,
1162                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1163                                                   NULL, "item not supported");
1164                 }
1165                 item_flags |= last_item;
1166         }
1167         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1168                 switch (actions->type) {
1169                 case RTE_FLOW_ACTION_TYPE_VOID:
1170                         break;
1171                 case RTE_FLOW_ACTION_TYPE_FLAG:
1172                         ret = mlx5_flow_validate_action_flag(action_flags,
1173                                                              attr,
1174                                                              error);
1175                         if (ret < 0)
1176                                 return ret;
1177                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1178                         break;
1179                 case RTE_FLOW_ACTION_TYPE_MARK:
1180                         ret = mlx5_flow_validate_action_mark(actions,
1181                                                              action_flags,
1182                                                              attr,
1183                                                              error);
1184                         if (ret < 0)
1185                                 return ret;
1186                         action_flags |= MLX5_FLOW_ACTION_MARK;
1187                         break;
1188                 case RTE_FLOW_ACTION_TYPE_DROP:
1189                         ret = mlx5_flow_validate_action_drop(action_flags,
1190                                                              attr,
1191                                                              error);
1192                         if (ret < 0)
1193                                 return ret;
1194                         action_flags |= MLX5_FLOW_ACTION_DROP;
1195                         break;
1196                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1197                         ret = mlx5_flow_validate_action_queue(actions,
1198                                                               action_flags, dev,
1199                                                               attr,
1200                                                               error);
1201                         if (ret < 0)
1202                                 return ret;
1203                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1204                         break;
1205                 case RTE_FLOW_ACTION_TYPE_RSS:
1206                         ret = mlx5_flow_validate_action_rss(actions,
1207                                                             action_flags, dev,
1208                                                             attr, item_flags,
1209                                                             error);
1210                         if (ret < 0)
1211                                 return ret;
1212                         action_flags |= MLX5_FLOW_ACTION_RSS;
1213                         break;
1214                 case RTE_FLOW_ACTION_TYPE_COUNT:
1215                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1216                         if (ret < 0)
1217                                 return ret;
1218                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1219                         break;
1220                 default:
1221                         return rte_flow_error_set(error, ENOTSUP,
1222                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1223                                                   actions,
1224                                                   "action not supported");
1225                 }
1226         }
1227         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1228                 return rte_flow_error_set(error, EINVAL,
1229                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1230                                           "no fate action is found");
1231         return 0;
1232 }
1233
1234 /**
1235  * Calculate the required bytes that are needed for the action part of the verbs
1236  * flow.
1237  *
1238  * @param[in] actions
1239  *   Pointer to the list of actions.
1240  *
1241  * @return
1242  *   The size of the memory needed for all actions.
1243  */
1244 static int
1245 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1246 {
1247         int size = 0;
1248
1249         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1250                 switch (actions->type) {
1251                 case RTE_FLOW_ACTION_TYPE_VOID:
1252                         break;
1253                 case RTE_FLOW_ACTION_TYPE_FLAG:
1254                         size += sizeof(struct ibv_flow_spec_action_tag);
1255                         break;
1256                 case RTE_FLOW_ACTION_TYPE_MARK:
1257                         size += sizeof(struct ibv_flow_spec_action_tag);
1258                         break;
1259                 case RTE_FLOW_ACTION_TYPE_DROP:
1260                         size += sizeof(struct ibv_flow_spec_action_drop);
1261                         break;
1262                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1263                         break;
1264                 case RTE_FLOW_ACTION_TYPE_RSS:
1265                         break;
1266                 case RTE_FLOW_ACTION_TYPE_COUNT:
1267 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1268         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1269                         size += sizeof(struct ibv_flow_spec_counter_action);
1270 #endif
1271                         break;
1272                 default:
1273                         break;
1274                 }
1275         }
1276         return size;
1277 }
1278
1279 /**
1280  * Calculate the required bytes that are needed for the item part of the verbs
1281  * flow.
1282  *
1283  * @param[in] items
1284  *   Pointer to the list of items.
1285  *
1286  * @return
1287  *   The size of the memory needed for all items.
1288  */
1289 static int
1290 flow_verbs_get_items_size(const struct rte_flow_item items[])
1291 {
1292         int size = 0;
1293
1294         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1295                 switch (items->type) {
1296                 case RTE_FLOW_ITEM_TYPE_VOID:
1297                         break;
1298                 case RTE_FLOW_ITEM_TYPE_ETH:
1299                         size += sizeof(struct ibv_flow_spec_eth);
1300                         break;
1301                 case RTE_FLOW_ITEM_TYPE_VLAN:
1302                         size += sizeof(struct ibv_flow_spec_eth);
1303                         break;
1304                 case RTE_FLOW_ITEM_TYPE_IPV4:
1305                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1306                         break;
1307                 case RTE_FLOW_ITEM_TYPE_IPV6:
1308                         size += sizeof(struct ibv_flow_spec_ipv6);
1309                         break;
1310                 case RTE_FLOW_ITEM_TYPE_UDP:
1311                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1312                         break;
1313                 case RTE_FLOW_ITEM_TYPE_TCP:
1314                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1315                         break;
1316                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1317                         size += sizeof(struct ibv_flow_spec_tunnel);
1318                         break;
1319                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1320                         size += sizeof(struct ibv_flow_spec_tunnel);
1321                         break;
1322 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1323                 case RTE_FLOW_ITEM_TYPE_GRE:
1324                         size += sizeof(struct ibv_flow_spec_gre);
1325                         break;
1326                 case RTE_FLOW_ITEM_TYPE_MPLS:
1327                         size += sizeof(struct ibv_flow_spec_mpls);
1328                         break;
1329 #else
1330                 case RTE_FLOW_ITEM_TYPE_GRE:
1331                         size += sizeof(struct ibv_flow_spec_tunnel);
1332                         break;
1333 #endif
1334                 default:
1335                         break;
1336                 }
1337         }
1338         return size;
1339 }
1340
1341 /**
1342  * Internal preparation function. Allocate mlx5_flow with the required size.
1343  * The required size is calculate based on the actions and items. This function
1344  * also returns the detected actions and items for later use.
1345  *
1346  * @param[in] attr
1347  *   Pointer to the flow attributes.
1348  * @param[in] items
1349  *   Pointer to the list of items.
1350  * @param[in] actions
1351  *   Pointer to the list of actions.
1352  * @param[out] error
1353  *   Pointer to the error structure.
1354  *
1355  * @return
1356  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1357  *   is set.
1358  */
1359 static struct mlx5_flow *
1360 flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
1361                    const struct rte_flow_item items[],
1362                    const struct rte_flow_action actions[],
1363                    struct rte_flow_error *error)
1364 {
1365         uint32_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
1366         struct mlx5_flow *flow;
1367
1368         size += flow_verbs_get_actions_size(actions);
1369         size += flow_verbs_get_items_size(items);
1370         flow = rte_calloc(__func__, 1, size, 0);
1371         if (!flow) {
1372                 rte_flow_error_set(error, ENOMEM,
1373                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1374                                    "not enough memory to create flow");
1375                 return NULL;
1376         }
1377         flow->verbs.attr = (void *)(flow + 1);
1378         flow->verbs.specs =
1379                 (uint8_t *)(flow + 1) + sizeof(struct ibv_flow_attr);
1380         return flow;
1381 }
1382
1383 /**
1384  * Fill the flow with verb spec.
1385  *
1386  * @param[in] dev
1387  *   Pointer to Ethernet device.
1388  * @param[in, out] dev_flow
1389  *   Pointer to the mlx5 flow.
1390  * @param[in] attr
1391  *   Pointer to the flow attributes.
1392  * @param[in] items
1393  *   Pointer to the list of items.
1394  * @param[in] actions
1395  *   Pointer to the list of actions.
1396  * @param[out] error
1397  *   Pointer to the error structure.
1398  *
1399  * @return
1400  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1401  */
1402 static int
1403 flow_verbs_translate(struct rte_eth_dev *dev,
1404                      struct mlx5_flow *dev_flow,
1405                      const struct rte_flow_attr *attr,
1406                      const struct rte_flow_item items[],
1407                      const struct rte_flow_action actions[],
1408                      struct rte_flow_error *error)
1409 {
1410         struct rte_flow *flow = dev_flow->flow;
1411         uint64_t item_flags = 0;
1412         uint64_t action_flags = 0;
1413         uint64_t priority = attr->priority;
1414         uint32_t subpriority = 0;
1415         struct mlx5_priv *priv = dev->data->dev_private;
1416
1417         if (priority == MLX5_FLOW_PRIO_RSVD)
1418                 priority = priv->config.flow_prio - 1;
1419         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1420                 int ret;
1421
1422                 switch (actions->type) {
1423                 case RTE_FLOW_ACTION_TYPE_VOID:
1424                         break;
1425                 case RTE_FLOW_ACTION_TYPE_FLAG:
1426                         flow_verbs_translate_action_flag(dev_flow, actions);
1427                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1428                         break;
1429                 case RTE_FLOW_ACTION_TYPE_MARK:
1430                         flow_verbs_translate_action_mark(dev_flow, actions);
1431                         action_flags |= MLX5_FLOW_ACTION_MARK;
1432                         break;
1433                 case RTE_FLOW_ACTION_TYPE_DROP:
1434                         flow_verbs_translate_action_drop(dev_flow, actions);
1435                         action_flags |= MLX5_FLOW_ACTION_DROP;
1436                         break;
1437                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1438                         flow_verbs_translate_action_queue(dev_flow, actions);
1439                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1440                         break;
1441                 case RTE_FLOW_ACTION_TYPE_RSS:
1442                         flow_verbs_translate_action_rss(dev_flow, actions);
1443                         action_flags |= MLX5_FLOW_ACTION_RSS;
1444                         break;
1445                 case RTE_FLOW_ACTION_TYPE_COUNT:
1446                         ret = flow_verbs_translate_action_count(dev_flow,
1447                                                                 actions,
1448                                                                 dev, error);
1449                         if (ret < 0)
1450                                 return ret;
1451                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1452                         break;
1453                 default:
1454                         return rte_flow_error_set(error, ENOTSUP,
1455                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1456                                                   actions,
1457                                                   "action not supported");
1458                 }
1459         }
1460         flow->actions = action_flags;
1461         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1462                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1463
1464                 switch (items->type) {
1465                 case RTE_FLOW_ITEM_TYPE_VOID:
1466                         break;
1467                 case RTE_FLOW_ITEM_TYPE_ETH:
1468                         flow_verbs_translate_item_eth(dev_flow, items,
1469                                                       item_flags);
1470                         subpriority = MLX5_PRIORITY_MAP_L2;
1471                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1472                                                MLX5_FLOW_LAYER_OUTER_L2;
1473                         break;
1474                 case RTE_FLOW_ITEM_TYPE_VLAN:
1475                         flow_verbs_translate_item_vlan(dev_flow, items,
1476                                                        item_flags);
1477                         subpriority = MLX5_PRIORITY_MAP_L2;
1478                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1479                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1480                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1481                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1482                         break;
1483                 case RTE_FLOW_ITEM_TYPE_IPV4:
1484                         flow_verbs_translate_item_ipv4(dev_flow, items,
1485                                                        item_flags);
1486                         subpriority = MLX5_PRIORITY_MAP_L3;
1487                         dev_flow->verbs.hash_fields |=
1488                                 mlx5_flow_hashfields_adjust
1489                                         (dev_flow, tunnel,
1490                                          MLX5_IPV4_LAYER_TYPES,
1491                                          MLX5_IPV4_IBV_RX_HASH);
1492                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1493                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1494                         break;
1495                 case RTE_FLOW_ITEM_TYPE_IPV6:
1496                         flow_verbs_translate_item_ipv6(dev_flow, items,
1497                                                        item_flags);
1498                         subpriority = MLX5_PRIORITY_MAP_L3;
1499                         dev_flow->verbs.hash_fields |=
1500                                 mlx5_flow_hashfields_adjust
1501                                         (dev_flow, tunnel,
1502                                          MLX5_IPV6_LAYER_TYPES,
1503                                          MLX5_IPV6_IBV_RX_HASH);
1504                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1505                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1506                         break;
1507                 case RTE_FLOW_ITEM_TYPE_TCP:
1508                         flow_verbs_translate_item_tcp(dev_flow, items,
1509                                                       item_flags);
1510                         subpriority = MLX5_PRIORITY_MAP_L4;
1511                         dev_flow->verbs.hash_fields |=
1512                                 mlx5_flow_hashfields_adjust
1513                                         (dev_flow, tunnel, ETH_RSS_TCP,
1514                                          (IBV_RX_HASH_SRC_PORT_TCP |
1515                                           IBV_RX_HASH_DST_PORT_TCP));
1516                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1517                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1518                         break;
1519                 case RTE_FLOW_ITEM_TYPE_UDP:
1520                         flow_verbs_translate_item_udp(dev_flow, items,
1521                                                       item_flags);
1522                         subpriority = MLX5_PRIORITY_MAP_L4;
1523                         dev_flow->verbs.hash_fields |=
1524                                 mlx5_flow_hashfields_adjust
1525                                         (dev_flow, tunnel, ETH_RSS_UDP,
1526                                          (IBV_RX_HASH_SRC_PORT_UDP |
1527                                           IBV_RX_HASH_DST_PORT_UDP));
1528                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1529                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1530                         break;
1531                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1532                         flow_verbs_translate_item_vxlan(dev_flow, items,
1533                                                         item_flags);
1534                         subpriority = MLX5_PRIORITY_MAP_L2;
1535                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1536                         break;
1537                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1538                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1539                                                             item_flags);
1540                         subpriority = MLX5_PRIORITY_MAP_L2;
1541                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1542                         break;
1543                 case RTE_FLOW_ITEM_TYPE_GRE:
1544                         flow_verbs_translate_item_gre(dev_flow, items,
1545                                                       item_flags);
1546                         subpriority = MLX5_PRIORITY_MAP_L2;
1547                         item_flags |= MLX5_FLOW_LAYER_GRE;
1548                         break;
1549                 case RTE_FLOW_ITEM_TYPE_MPLS:
1550                         flow_verbs_translate_item_mpls(dev_flow, items,
1551                                                        item_flags);
1552                         subpriority = MLX5_PRIORITY_MAP_L2;
1553                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1554                         break;
1555                 default:
1556                         return rte_flow_error_set(error, ENOTSUP,
1557                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1558                                                   NULL,
1559                                                   "item not supported");
1560                 }
1561         }
1562         dev_flow->layers = item_flags;
1563         dev_flow->verbs.attr->priority =
1564                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1565         dev_flow->verbs.attr->port = (uint8_t)priv->ibv_port;
1566         return 0;
1567 }
1568
1569 /**
1570  * Remove the flow from the NIC but keeps it in memory.
1571  *
1572  * @param[in] dev
1573  *   Pointer to the Ethernet device structure.
1574  * @param[in, out] flow
1575  *   Pointer to flow structure.
1576  */
1577 static void
1578 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1579 {
1580         struct mlx5_flow_verbs *verbs;
1581         struct mlx5_flow *dev_flow;
1582
1583         if (!flow)
1584                 return;
1585         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1586                 verbs = &dev_flow->verbs;
1587                 if (verbs->flow) {
1588                         claim_zero(mlx5_glue->destroy_flow(verbs->flow));
1589                         verbs->flow = NULL;
1590                 }
1591                 if (verbs->hrxq) {
1592                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1593                                 mlx5_hrxq_drop_release(dev);
1594                         else
1595                                 mlx5_hrxq_release(dev, verbs->hrxq);
1596                         verbs->hrxq = NULL;
1597                 }
1598                 if (dev_flow->verbs.vf_vlan.tag &&
1599                     dev_flow->verbs.vf_vlan.created) {
1600                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1601                 }
1602         }
1603 }
1604
1605 /**
1606  * Remove the flow from the NIC and the memory.
1607  *
1608  * @param[in] dev
1609  *   Pointer to the Ethernet device structure.
1610  * @param[in, out] flow
1611  *   Pointer to flow structure.
1612  */
1613 static void
1614 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1615 {
1616         struct mlx5_flow *dev_flow;
1617
1618         if (!flow)
1619                 return;
1620         flow_verbs_remove(dev, flow);
1621         while (!LIST_EMPTY(&flow->dev_flows)) {
1622                 dev_flow = LIST_FIRST(&flow->dev_flows);
1623                 LIST_REMOVE(dev_flow, next);
1624                 rte_free(dev_flow);
1625         }
1626         if (flow->counter) {
1627                 flow_verbs_counter_release(dev, flow->counter);
1628                 flow->counter = NULL;
1629         }
1630 }
1631
1632 /**
1633  * Apply the flow to the NIC.
1634  *
1635  * @param[in] dev
1636  *   Pointer to the Ethernet device structure.
1637  * @param[in, out] flow
1638  *   Pointer to flow structure.
1639  * @param[out] error
1640  *   Pointer to error structure.
1641  *
1642  * @return
1643  *   0 on success, a negative errno value otherwise and rte_errno is set.
1644  */
1645 static int
1646 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1647                  struct rte_flow_error *error)
1648 {
1649         struct mlx5_priv *priv = dev->data->dev_private;
1650         struct mlx5_flow_verbs *verbs;
1651         struct mlx5_flow *dev_flow;
1652         int err;
1653
1654         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1655                 verbs = &dev_flow->verbs;
1656                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1657                         verbs->hrxq = mlx5_hrxq_drop_new(dev);
1658                         if (!verbs->hrxq) {
1659                                 rte_flow_error_set
1660                                         (error, errno,
1661                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1662                                          "cannot get drop hash queue");
1663                                 goto error;
1664                         }
1665                 } else {
1666                         struct mlx5_hrxq *hrxq;
1667
1668                         hrxq = mlx5_hrxq_get(dev, flow->key,
1669                                              MLX5_RSS_HASH_KEY_LEN,
1670                                              verbs->hash_fields,
1671                                              (*flow->queue),
1672                                              flow->rss.queue_num);
1673                         if (!hrxq)
1674                                 hrxq = mlx5_hrxq_new(dev, flow->key,
1675                                                      MLX5_RSS_HASH_KEY_LEN,
1676                                                      verbs->hash_fields,
1677                                                      (*flow->queue),
1678                                                      flow->rss.queue_num,
1679                                                      !!(dev_flow->layers &
1680                                                        MLX5_FLOW_LAYER_TUNNEL));
1681                         if (!hrxq) {
1682                                 rte_flow_error_set
1683                                         (error, rte_errno,
1684                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1685                                          "cannot get hash queue");
1686                                 goto error;
1687                         }
1688                         verbs->hrxq = hrxq;
1689                 }
1690                 verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp,
1691                                                      verbs->attr);
1692                 if (!verbs->flow) {
1693                         rte_flow_error_set(error, errno,
1694                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1695                                            NULL,
1696                                            "hardware refuses to create flow");
1697                         goto error;
1698                 }
1699                 if (priv->vmwa_context &&
1700                     dev_flow->verbs.vf_vlan.tag &&
1701                     !dev_flow->verbs.vf_vlan.created) {
1702                         /*
1703                          * The rule contains the VLAN pattern.
1704                          * For VF we are going to create VLAN
1705                          * interface to make hypervisor set correct
1706                          * e-Switch vport context.
1707                          */
1708                         mlx5_vlan_vmwa_acquire(dev, &dev_flow->verbs.vf_vlan);
1709                 }
1710         }
1711         return 0;
1712 error:
1713         err = rte_errno; /* Save rte_errno before cleanup. */
1714         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1715                 verbs = &dev_flow->verbs;
1716                 if (verbs->hrxq) {
1717                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1718                                 mlx5_hrxq_drop_release(dev);
1719                         else
1720                                 mlx5_hrxq_release(dev, verbs->hrxq);
1721                         verbs->hrxq = NULL;
1722                 }
1723                 if (dev_flow->verbs.vf_vlan.tag &&
1724                     dev_flow->verbs.vf_vlan.created) {
1725                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1726                 }
1727         }
1728         rte_errno = err; /* Restore rte_errno. */
1729         return -rte_errno;
1730 }
1731
1732 /**
1733  * Query a flow.
1734  *
1735  * @see rte_flow_query()
1736  * @see rte_flow_ops
1737  */
1738 static int
1739 flow_verbs_query(struct rte_eth_dev *dev,
1740                  struct rte_flow *flow,
1741                  const struct rte_flow_action *actions,
1742                  void *data,
1743                  struct rte_flow_error *error)
1744 {
1745         int ret = -EINVAL;
1746
1747         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1748                 switch (actions->type) {
1749                 case RTE_FLOW_ACTION_TYPE_VOID:
1750                         break;
1751                 case RTE_FLOW_ACTION_TYPE_COUNT:
1752                         ret = flow_verbs_counter_query(dev, flow, data, error);
1753                         break;
1754                 default:
1755                         return rte_flow_error_set(error, ENOTSUP,
1756                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1757                                                   actions,
1758                                                   "action not supported");
1759                 }
1760         }
1761         return ret;
1762 }
1763
1764 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1765         .validate = flow_verbs_validate,
1766         .prepare = flow_verbs_prepare,
1767         .translate = flow_verbs_translate,
1768         .apply = flow_verbs_apply,
1769         .remove = flow_verbs_remove,
1770         .destroy = flow_verbs_destroy,
1771         .query = flow_verbs_query,
1772 };