drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28
  29 #include "mlx5.h"
  30 #include "mlx5_defs.h"
  31 #include "mlx5_flow.h"
  32 #include "mlx5_glue.h"
  33 #include "mlx5_prm.h"
  34 #include "mlx5_rxtx.h"
  35
  36 #define VERBS_SPEC_INNER(item_flags) \
  37         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  38
  39 /**
  40  * Create Verbs flow counter with Verbs library.
  41  *
  42  * @param[in] dev
  43  *   Pointer to the Ethernet device structure.
  44  * @param[in, out] counter
  45  *   mlx5 flow counter object, contains the counter id,
  46  *   handle of created Verbs flow counter is returned
  47  *   in cs field (if counters are supported).
  48  *
  49  * @return
  50  *   0 On success else a negative errno value is returned
  51  *   and rte_errno is set.
  52  */
  53 static int
  54 flow_verbs_counter_create(struct rte_eth_dev *dev,
  55                           struct mlx5_flow_counter *counter)
  56 {
  57 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  58         struct mlx5_priv *priv = dev->data->dev_private;
  59         struct ibv_context *ctx = priv->sh->ctx;
  60         struct ibv_counter_set_init_attr init = {
  61                          .counter_set_id = counter->id};
  62
  63         counter->cs = mlx5_glue->create_counter_set(ctx, &init);
  64         if (!counter->cs) {
  65                 rte_errno = ENOTSUP;
  66                 return -ENOTSUP;
  67         }
  68         return 0;
  69 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
  70         struct mlx5_priv *priv = dev->data->dev_private;
  71         struct ibv_context *ctx = priv->sh->ctx;
  72         struct ibv_counters_init_attr init = {0};
  73         struct ibv_counter_attach_attr attach;
  74         int ret;
  75
  76         memset(&attach, 0, sizeof(attach));
  77         counter->cs = mlx5_glue->create_counters(ctx, &init);
  78         if (!counter->cs) {
  79                 rte_errno = ENOTSUP;
  80                 return -ENOTSUP;
  81         }
  82         attach.counter_desc = IBV_COUNTER_PACKETS;
  83         attach.index = 0;
  84         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
  85         if (!ret) {
  86                 attach.counter_desc = IBV_COUNTER_BYTES;
  87                 attach.index = 1;
  88                 ret = mlx5_glue->attach_counters
  89                                         (counter->cs, &attach, NULL);
  90         }
  91         if (ret) {
  92                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
  93                 counter->cs = NULL;
  94                 rte_errno = ret;
  95                 return -ret;
  96         }
  97         return 0;
  98 #else
  99         (void)dev;
 100         (void)counter;
 101         rte_errno = ENOTSUP;
 102         return -ENOTSUP;
 103 #endif
 104 }
 105
 106 /**
 107  * Get a flow counter.
 108  *
 109  * @param[in] dev
 110  *   Pointer to the Ethernet device structure.
 111  * @param[in] shared
 112  *   Indicate if this counter is shared with other flows.
 113  * @param[in] id
 114  *   Counter identifier.
 115  *
 116  * @return
 117  *   A pointer to the counter, NULL otherwise and rte_errno is set.
 118  */
 119 static struct mlx5_flow_counter *
 120 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 121 {
 122         struct mlx5_priv *priv = dev->data->dev_private;
 123         struct mlx5_flow_counter *cnt;
 124         int ret;
 125
 126         if (shared) {
 127                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
 128                         if (cnt->shared && cnt->id == id) {
 129                                 cnt->ref_cnt++;
 130                                 return cnt;
 131                         }
 132                 }
 133         }
 134         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 135         if (!cnt) {
 136                 rte_errno = ENOMEM;
 137                 return NULL;
 138         }
 139         cnt->id = id;
 140         cnt->shared = shared;
 141         cnt->ref_cnt = 1;
 142         cnt->hits = 0;
 143         cnt->bytes = 0;
 144         /* Create counter with Verbs. */
 145         ret = flow_verbs_counter_create(dev, cnt);
 146         if (!ret) {
 147                 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
 148                 return cnt;
 149         }
 150         /* Some error occurred in Verbs library. */
 151         rte_free(cnt);
 152         rte_errno = -ret;
 153         return NULL;
 154 }
 155
 156 /**
 157  * Release a flow counter.
 158  *
 159  * @param[in] dev
 160  *   Pointer to the Ethernet device structure.
 161  * @param[in] counter
 162  *   Pointer to the counter handler.
 163  */
 164 static void
 165 flow_verbs_counter_release(struct rte_eth_dev *dev,
 166                            struct mlx5_flow_counter *counter)
 167 {
 168         struct mlx5_priv *priv = dev->data->dev_private;
 169
 170         if (--counter->ref_cnt == 0) {
 171 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 172                 claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
 173 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 174                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 175 #endif
 176                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
 177                 rte_free(counter);
 178         }
 179 }
 180
 181 /**
 182  * Query a flow counter via Verbs library call.
 183  *
 184  * @see rte_flow_query()
 185  * @see rte_flow_ops
 186  */
 187 static int
 188 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 189                          struct rte_flow *flow, void *data,
 190                          struct rte_flow_error *error)
 191 {
 192 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 193         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 194         if (flow->counter->cs) {
 195                 struct rte_flow_query_count *qc = data;
 196                 uint64_t counters[2] = {0, 0};
 197 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 198                 struct ibv_query_counter_set_attr query_cs_attr = {
 199                         .cs = flow->counter->cs,
 200                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 201                 };
 202                 struct ibv_counter_set_data query_out = {
 203                         .out = counters,
 204                         .outlen = 2 * sizeof(uint64_t),
 205                 };
 206                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 207                                                        &query_out);
 208 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 209                 int err = mlx5_glue->query_counters
 210                                (flow->counter->cs, counters,
 211                                 RTE_DIM(counters),
 212                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 213 #endif
 214                 if (err)
 215                         return rte_flow_error_set
 216                                 (error, err,
 217                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 218                                  NULL,
 219                                  "cannot read counter");
 220                 qc->hits_set = 1;
 221                 qc->bytes_set = 1;
 222                 qc->hits = counters[0] - flow->counter->hits;
 223                 qc->bytes = counters[1] - flow->counter->bytes;
 224                 if (qc->reset) {
 225                         flow->counter->hits = counters[0];
 226                         flow->counter->bytes = counters[1];
 227                 }
 228                 return 0;
 229         }
 230         return rte_flow_error_set(error, EINVAL,
 231                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 232                                   NULL,
 233                                   "flow does not have counter");
 234 #else
 235         (void)flow;
 236         (void)data;
 237         return rte_flow_error_set(error, ENOTSUP,
 238                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 239                                   NULL,
 240                                   "counters are not available");
 241 #endif
 242 }
 243
 244 /**
 245  * Add a verbs item specification into @p verbs.
 246  *
 247  * @param[out] verbs
 248  *   Pointer to verbs structure.
 249  * @param[in] src
 250  *   Create specification.
 251  * @param[in] size
 252  *   Size in bytes of the specification to copy.
 253  */
 254 static void
 255 flow_verbs_spec_add(struct mlx5_flow_verbs *verbs, void *src, unsigned int size)
 256 {
 257         void *dst;
 258
 259         if (!verbs)
 260                 return;
 261         assert(verbs->specs);
 262         dst = (void *)(verbs->specs + verbs->size);
 263         memcpy(dst, src, size);
 264         ++verbs->attr->num_of_specs;
 265         verbs->size += size;
 266 }
 267
 268 /**
 269  * Convert the @p item into a Verbs specification. This function assumes that
 270  * the input is valid and that there is space to insert the requested item
 271  * into the flow.
 272  *
 273  * @param[in, out] dev_flow
 274  *   Pointer to dev_flow structure.
 275  * @param[in] item
 276  *   Item specification.
 277  * @param[in] item_flags
 278  *   Parsed item flags.
 279  */
 280 static void
 281 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 282                               const struct rte_flow_item *item,
 283                               uint64_t item_flags)
 284 {
 285         const struct rte_flow_item_eth *spec = item->spec;
 286         const struct rte_flow_item_eth *mask = item->mask;
 287         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 288         struct ibv_flow_spec_eth eth = {
 289                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 290                 .size = size,
 291         };
 292
 293         if (!mask)
 294                 mask = &rte_flow_item_eth_mask;
 295         if (spec) {
 296                 unsigned int i;
 297
 298                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 299                         RTE_ETHER_ADDR_LEN);
 300                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 301                         RTE_ETHER_ADDR_LEN);
 302                 eth.val.ether_type = spec->type;
 303                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 304                         RTE_ETHER_ADDR_LEN);
 305                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 306                         RTE_ETHER_ADDR_LEN);
 307                 eth.mask.ether_type = mask->type;
 308                 /* Remove unwanted bits from values. */
 309                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 310                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 311                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 312                 }
 313                 eth.val.ether_type &= eth.mask.ether_type;
 314         }
 315         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 316 }
 317
 318 /**
 319  * Update the VLAN tag in the Verbs Ethernet specification.
 320  * This function assumes that the input is valid and there is space to add
 321  * the requested item.
 322  *
 323  * @param[in, out] attr
 324  *   Pointer to Verbs attributes structure.
 325  * @param[in] eth
 326  *   Verbs structure containing the VLAN information to copy.
 327  */
 328 static void
 329 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 330                             struct ibv_flow_spec_eth *eth)
 331 {
 332         unsigned int i;
 333         const enum ibv_flow_spec_type search = eth->type;
 334         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 335                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 336
 337         for (i = 0; i != attr->num_of_specs; ++i) {
 338                 if (hdr->type == search) {
 339                         struct ibv_flow_spec_eth *e =
 340                                 (struct ibv_flow_spec_eth *)hdr;
 341
 342                         e->val.vlan_tag = eth->val.vlan_tag;
 343                         e->mask.vlan_tag = eth->mask.vlan_tag;
 344                         e->val.ether_type = eth->val.ether_type;
 345                         e->mask.ether_type = eth->mask.ether_type;
 346                         break;
 347                 }
 348                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 349         }
 350 }
 351
 352 /**
 353  * Convert the @p item into a Verbs specification. This function assumes that
 354  * the input is valid and that there is space to insert the requested item
 355  * into the flow.
 356  *
 357  * @param[in, out] dev_flow
 358  *   Pointer to dev_flow structure.
 359  * @param[in] item
 360  *   Item specification.
 361  * @param[in] item_flags
 362  *   Parsed item flags.
 363  */
 364 static void
 365 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 366                                const struct rte_flow_item *item,
 367                                uint64_t item_flags)
 368 {
 369         const struct rte_flow_item_vlan *spec = item->spec;
 370         const struct rte_flow_item_vlan *mask = item->mask;
 371         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 372         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 373         struct ibv_flow_spec_eth eth = {
 374                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 375                 .size = size,
 376         };
 377         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 378                                       MLX5_FLOW_LAYER_OUTER_L2;
 379
 380         if (!mask)
 381                 mask = &rte_flow_item_vlan_mask;
 382         if (spec) {
 383                 eth.val.vlan_tag = spec->tci;
 384                 eth.mask.vlan_tag = mask->tci;
 385                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 386                 eth.val.ether_type = spec->inner_type;
 387                 eth.mask.ether_type = mask->inner_type;
 388                 eth.val.ether_type &= eth.mask.ether_type;
 389         }
 390         if (!(item_flags & l2m))
 391                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 392         else
 393                 flow_verbs_item_vlan_update(dev_flow->verbs.attr, &eth);
 394         if (!tunnel)
 395                 dev_flow->verbs.vf_vlan.tag =
 396                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 397 }
 398
 399 /**
 400  * Convert the @p item into a Verbs specification. This function assumes that
 401  * the input is valid and that there is space to insert the requested item
 402  * into the flow.
 403  *
 404  * @param[in, out] dev_flow
 405  *   Pointer to dev_flow structure.
 406  * @param[in] item
 407  *   Item specification.
 408  * @param[in] item_flags
 409  *   Parsed item flags.
 410  */
 411 static void
 412 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 413                                const struct rte_flow_item *item,
 414                                uint64_t item_flags)
 415 {
 416         const struct rte_flow_item_ipv4 *spec = item->spec;
 417         const struct rte_flow_item_ipv4 *mask = item->mask;
 418         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 419         struct ibv_flow_spec_ipv4_ext ipv4 = {
 420                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 421                 .size = size,
 422         };
 423
 424         if (!mask)
 425                 mask = &rte_flow_item_ipv4_mask;
 426         if (spec) {
 427                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 428                         .src_ip = spec->hdr.src_addr,
 429                         .dst_ip = spec->hdr.dst_addr,
 430                         .proto = spec->hdr.next_proto_id,
 431                         .tos = spec->hdr.type_of_service,
 432                 };
 433                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 434                         .src_ip = mask->hdr.src_addr,
 435                         .dst_ip = mask->hdr.dst_addr,
 436                         .proto = mask->hdr.next_proto_id,
 437                         .tos = mask->hdr.type_of_service,
 438                 };
 439                 /* Remove unwanted bits from values. */
 440                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 441                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 442                 ipv4.val.proto &= ipv4.mask.proto;
 443                 ipv4.val.tos &= ipv4.mask.tos;
 444         }
 445         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 446 }
 447
 448 /**
 449  * Convert the @p item into a Verbs specification. This function assumes that
 450  * the input is valid and that there is space to insert the requested item
 451  * into the flow.
 452  *
 453  * @param[in, out] dev_flow
 454  *   Pointer to dev_flow structure.
 455  * @param[in] item
 456  *   Item specification.
 457  * @param[in] item_flags
 458  *   Parsed item flags.
 459  */
 460 static void
 461 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 462                                const struct rte_flow_item *item,
 463                                uint64_t item_flags)
 464 {
 465         const struct rte_flow_item_ipv6 *spec = item->spec;
 466         const struct rte_flow_item_ipv6 *mask = item->mask;
 467         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 468         struct ibv_flow_spec_ipv6 ipv6 = {
 469                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 470                 .size = size,
 471         };
 472
 473         if (!mask)
 474                 mask = &rte_flow_item_ipv6_mask;
 475         if (spec) {
 476                 unsigned int i;
 477                 uint32_t vtc_flow_val;
 478                 uint32_t vtc_flow_mask;
 479
 480                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 481                        RTE_DIM(ipv6.val.src_ip));
 482                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 483                        RTE_DIM(ipv6.val.dst_ip));
 484                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 485                        RTE_DIM(ipv6.mask.src_ip));
 486                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 487                        RTE_DIM(ipv6.mask.dst_ip));
 488                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 489                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 490                 ipv6.val.flow_label =
 491                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 492                                          RTE_IPV6_HDR_FL_SHIFT);
 493                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 494                                          RTE_IPV6_HDR_TC_SHIFT;
 495                 ipv6.val.next_hdr = spec->hdr.proto;
 496                 ipv6.val.hop_limit = spec->hdr.hop_limits;
 497                 ipv6.mask.flow_label =
 498                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 499                                          RTE_IPV6_HDR_FL_SHIFT);
 500                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 501                                           RTE_IPV6_HDR_TC_SHIFT;
 502                 ipv6.mask.next_hdr = mask->hdr.proto;
 503                 ipv6.mask.hop_limit = mask->hdr.hop_limits;
 504                 /* Remove unwanted bits from values. */
 505                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 506                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 507                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 508                 }
 509                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 510                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 511                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 512                 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 513         }
 514         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 515 }
 516
 517 /**
 518  * Convert the @p item into a Verbs specification. This function assumes that
 519  * the input is valid and that there is space to insert the requested item
 520  * into the flow.
 521  *
 522  * @param[in, out] dev_flow
 523  *   Pointer to dev_flow structure.
 524  * @param[in] item
 525  *   Item specification.
 526  * @param[in] item_flags
 527  *   Parsed item flags.
 528  */
 529 static void
 530 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 531                               const struct rte_flow_item *item,
 532                               uint64_t item_flags __rte_unused)
 533 {
 534         const struct rte_flow_item_tcp *spec = item->spec;
 535         const struct rte_flow_item_tcp *mask = item->mask;
 536         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 537         struct ibv_flow_spec_tcp_udp tcp = {
 538                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 539                 .size = size,
 540         };
 541
 542         if (!mask)
 543                 mask = &rte_flow_item_tcp_mask;
 544         if (spec) {
 545                 tcp.val.dst_port = spec->hdr.dst_port;
 546                 tcp.val.src_port = spec->hdr.src_port;
 547                 tcp.mask.dst_port = mask->hdr.dst_port;
 548                 tcp.mask.src_port = mask->hdr.src_port;
 549                 /* Remove unwanted bits from values. */
 550                 tcp.val.src_port &= tcp.mask.src_port;
 551                 tcp.val.dst_port &= tcp.mask.dst_port;
 552         }
 553         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 554 }
 555
 556 /**
 557  * Convert the @p item into a Verbs specification. This function assumes that
 558  * the input is valid and that there is space to insert the requested item
 559  * into the flow.
 560  *
 561  * @param[in, out] dev_flow
 562  *   Pointer to dev_flow structure.
 563  * @param[in] item
 564  *   Item specification.
 565  * @param[in] item_flags
 566  *   Parsed item flags.
 567  */
 568 static void
 569 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 570                               const struct rte_flow_item *item,
 571                               uint64_t item_flags __rte_unused)
 572 {
 573         const struct rte_flow_item_udp *spec = item->spec;
 574         const struct rte_flow_item_udp *mask = item->mask;
 575         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 576         struct ibv_flow_spec_tcp_udp udp = {
 577                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 578                 .size = size,
 579         };
 580
 581         if (!mask)
 582                 mask = &rte_flow_item_udp_mask;
 583         if (spec) {
 584                 udp.val.dst_port = spec->hdr.dst_port;
 585                 udp.val.src_port = spec->hdr.src_port;
 586                 udp.mask.dst_port = mask->hdr.dst_port;
 587                 udp.mask.src_port = mask->hdr.src_port;
 588                 /* Remove unwanted bits from values. */
 589                 udp.val.src_port &= udp.mask.src_port;
 590                 udp.val.dst_port &= udp.mask.dst_port;
 591         }
 592         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 593 }
 594
 595 /**
 596  * Convert the @p item into a Verbs specification. This function assumes that
 597  * the input is valid and that there is space to insert the requested item
 598  * into the flow.
 599  *
 600  * @param[in, out] dev_flow
 601  *   Pointer to dev_flow structure.
 602  * @param[in] item
 603  *   Item specification.
 604  * @param[in] item_flags
 605  *   Parsed item flags.
 606  */
 607 static void
 608 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 609                                 const struct rte_flow_item *item,
 610                                 uint64_t item_flags __rte_unused)
 611 {
 612         const struct rte_flow_item_vxlan *spec = item->spec;
 613         const struct rte_flow_item_vxlan *mask = item->mask;
 614         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 615         struct ibv_flow_spec_tunnel vxlan = {
 616                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 617                 .size = size,
 618         };
 619         union vni {
 620                 uint32_t vlan_id;
 621                 uint8_t vni[4];
 622         } id = { .vlan_id = 0, };
 623
 624         if (!mask)
 625                 mask = &rte_flow_item_vxlan_mask;
 626         if (spec) {
 627                 memcpy(&id.vni[1], spec->vni, 3);
 628                 vxlan.val.tunnel_id = id.vlan_id;
 629                 memcpy(&id.vni[1], mask->vni, 3);
 630                 vxlan.mask.tunnel_id = id.vlan_id;
 631                 /* Remove unwanted bits from values. */
 632                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 633         }
 634         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 635 }
 636
 637 /**
 638  * Convert the @p item into a Verbs specification. This function assumes that
 639  * the input is valid and that there is space to insert the requested item
 640  * into the flow.
 641  *
 642  * @param[in, out] dev_flow
 643  *   Pointer to dev_flow structure.
 644  * @param[in] item
 645  *   Item specification.
 646  * @param[in] item_flags
 647  *   Parsed item flags.
 648  */
 649 static void
 650 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 651                                     const struct rte_flow_item *item,
 652                                     uint64_t item_flags __rte_unused)
 653 {
 654         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 655         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 656         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 657         struct ibv_flow_spec_tunnel vxlan_gpe = {
 658                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 659                 .size = size,
 660         };
 661         union vni {
 662                 uint32_t vlan_id;
 663                 uint8_t vni[4];
 664         } id = { .vlan_id = 0, };
 665
 666         if (!mask)
 667                 mask = &rte_flow_item_vxlan_gpe_mask;
 668         if (spec) {
 669                 memcpy(&id.vni[1], spec->vni, 3);
 670                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 671                 memcpy(&id.vni[1], mask->vni, 3);
 672                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 673                 /* Remove unwanted bits from values. */
 674                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 675         }
 676         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 677 }
 678
 679 /**
 680  * Update the protocol in Verbs IPv4/IPv6 spec.
 681  *
 682  * @param[in, out] attr
 683  *   Pointer to Verbs attributes structure.
 684  * @param[in] search
 685  *   Specification type to search in order to update the IP protocol.
 686  * @param[in] protocol
 687  *   Protocol value to set if none is present in the specification.
 688  */
 689 static void
 690 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 691                                        enum ibv_flow_spec_type search,
 692                                        uint8_t protocol)
 693 {
 694         unsigned int i;
 695         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 696                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 697
 698         if (!attr)
 699                 return;
 700         for (i = 0; i != attr->num_of_specs; ++i) {
 701                 if (hdr->type == search) {
 702                         union {
 703                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 704                                 struct ibv_flow_spec_ipv6 *ipv6;
 705                         } ip;
 706
 707                         switch (search) {
 708                         case IBV_FLOW_SPEC_IPV4_EXT:
 709                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 710                                 if (!ip.ipv4->val.proto) {
 711                                         ip.ipv4->val.proto = protocol;
 712                                         ip.ipv4->mask.proto = 0xff;
 713                                 }
 714                                 break;
 715                         case IBV_FLOW_SPEC_IPV6:
 716                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 717                                 if (!ip.ipv6->val.next_hdr) {
 718                                         ip.ipv6->val.next_hdr = protocol;
 719                                         ip.ipv6->mask.next_hdr = 0xff;
 720                                 }
 721                                 break;
 722                         default:
 723                                 break;
 724                         }
 725                         break;
 726                 }
 727                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 728         }
 729 }
 730
 731 /**
 732  * Convert the @p item into a Verbs specification. This function assumes that
 733  * the input is valid and that there is space to insert the requested item
 734  * into the flow.
 735  *
 736  * @param[in, out] dev_flow
 737  *   Pointer to dev_flow structure.
 738  * @param[in] item
 739  *   Item specification.
 740  * @param[in] item_flags
 741  *   Parsed item flags.
 742  */
 743 static void
 744 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 745                               const struct rte_flow_item *item __rte_unused,
 746                               uint64_t item_flags)
 747 {
 748         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
 749 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 750         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 751         struct ibv_flow_spec_tunnel tunnel = {
 752                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 753                 .size = size,
 754         };
 755 #else
 756         const struct rte_flow_item_gre *spec = item->spec;
 757         const struct rte_flow_item_gre *mask = item->mask;
 758         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 759         struct ibv_flow_spec_gre tunnel = {
 760                 .type = IBV_FLOW_SPEC_GRE,
 761                 .size = size,
 762         };
 763
 764         if (!mask)
 765                 mask = &rte_flow_item_gre_mask;
 766         if (spec) {
 767                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 768                 tunnel.val.protocol = spec->protocol;
 769                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 770                 tunnel.mask.protocol = mask->protocol;
 771                 /* Remove unwanted bits from values. */
 772                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 773                 tunnel.val.protocol &= tunnel.mask.protocol;
 774                 tunnel.val.key &= tunnel.mask.key;
 775         }
 776 #endif
 777         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 778                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 779                                                        IBV_FLOW_SPEC_IPV4_EXT,
 780                                                        IPPROTO_GRE);
 781         else
 782                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 783                                                        IBV_FLOW_SPEC_IPV6,
 784                                                        IPPROTO_GRE);
 785         flow_verbs_spec_add(verbs, &tunnel, size);
 786 }
 787
 788 /**
 789  * Convert the @p action into a Verbs specification. This function assumes that
 790  * the input is valid and that there is space to insert the requested action
 791  * into the flow. This function also return the action that was added.
 792  *
 793  * @param[in, out] dev_flow
 794  *   Pointer to dev_flow structure.
 795  * @param[in] item
 796  *   Item specification.
 797  * @param[in] item_flags
 798  *   Parsed item flags.
 799  */
 800 static void
 801 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
 802                                const struct rte_flow_item *item __rte_unused,
 803                                uint64_t item_flags __rte_unused)
 804 {
 805 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 806         const struct rte_flow_item_mpls *spec = item->spec;
 807         const struct rte_flow_item_mpls *mask = item->mask;
 808         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 809         struct ibv_flow_spec_mpls mpls = {
 810                 .type = IBV_FLOW_SPEC_MPLS,
 811                 .size = size,
 812         };
 813
 814         if (!mask)
 815                 mask = &rte_flow_item_mpls_mask;
 816         if (spec) {
 817                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 818                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 819                 /* Remove unwanted bits from values.  */
 820                 mpls.val.label &= mpls.mask.label;
 821         }
 822         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
 823 #endif
 824 }
 825
 826 /**
 827  * Convert the @p action into a Verbs specification. This function assumes that
 828  * the input is valid and that there is space to insert the requested action
 829  * into the flow.
 830  *
 831  * @param[in] dev_flow
 832  *   Pointer to mlx5_flow.
 833  * @param[in] action
 834  *   Action configuration.
 835  */
 836 static void
 837 flow_verbs_translate_action_drop
 838         (struct mlx5_flow *dev_flow,
 839          const struct rte_flow_action *action __rte_unused)
 840 {
 841         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 842         struct ibv_flow_spec_action_drop drop = {
 843                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 844                         .size = size,
 845         };
 846
 847         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
 848 }
 849
 850 /**
 851  * Convert the @p action into a Verbs specification. This function assumes that
 852  * the input is valid and that there is space to insert the requested action
 853  * into the flow.
 854  *
 855  * @param[in] dev_flow
 856  *   Pointer to mlx5_flow.
 857  * @param[in] action
 858  *   Action configuration.
 859  */
 860 static void
 861 flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow,
 862                                   const struct rte_flow_action *action)
 863 {
 864         const struct rte_flow_action_queue *queue = action->conf;
 865         struct rte_flow *flow = dev_flow->flow;
 866
 867         if (flow->queue)
 868                 (*flow->queue)[0] = queue->index;
 869         flow->rss.queue_num = 1;
 870 }
 871
 872 /**
 873  * Convert the @p action into a Verbs specification. This function assumes that
 874  * the input is valid and that there is space to insert the requested action
 875  * into the flow.
 876  *
 877  * @param[in] action
 878  *   Action configuration.
 879  * @param[in, out] action_flags
 880  *   Pointer to the detected actions.
 881  * @param[in] dev_flow
 882  *   Pointer to mlx5_flow.
 883  */
 884 static void
 885 flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow,
 886                                 const struct rte_flow_action *action)
 887 {
 888         const struct rte_flow_action_rss *rss = action->conf;
 889         const uint8_t *rss_key;
 890         struct rte_flow *flow = dev_flow->flow;
 891
 892         if (flow->queue)
 893                 memcpy((*flow->queue), rss->queue,
 894                        rss->queue_num * sizeof(uint16_t));
 895         flow->rss.queue_num = rss->queue_num;
 896         /* NULL RSS key indicates default RSS key. */
 897         rss_key = !rss->key ? rss_hash_default_key : rss->key;
 898         memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
 899         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
 900         flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
 901         flow->rss.level = rss->level;
 902 }
 903
 904 /**
 905  * Convert the @p action into a Verbs specification. This function assumes that
 906  * the input is valid and that there is space to insert the requested action
 907  * into the flow.
 908  *
 909  * @param[in] dev_flow
 910  *   Pointer to mlx5_flow.
 911  * @param[in] action
 912  *   Action configuration.
 913  */
 914 static void
 915 flow_verbs_translate_action_flag
 916         (struct mlx5_flow *dev_flow,
 917          const struct rte_flow_action *action __rte_unused)
 918 {
 919         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 920         struct ibv_flow_spec_action_tag tag = {
 921                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 922                 .size = size,
 923                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 924         };
 925
 926         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 927 }
 928
 929 /**
 930  * Convert the @p action into a Verbs specification. This function assumes that
 931  * the input is valid and that there is space to insert the requested action
 932  * into the flow.
 933  *
 934  * @param[in] dev_flow
 935  *   Pointer to mlx5_flow.
 936  * @param[in] action
 937  *   Action configuration.
 938  */
 939 static void
 940 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
 941                                  const struct rte_flow_action *action)
 942 {
 943         const struct rte_flow_action_mark *mark = action->conf;
 944         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 945         struct ibv_flow_spec_action_tag tag = {
 946                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 947                 .size = size,
 948                 .tag_id = mlx5_flow_mark_set(mark->id),
 949         };
 950
 951         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 952 }
 953
 954 /**
 955  * Convert the @p action into a Verbs specification. This function assumes that
 956  * the input is valid and that there is space to insert the requested action
 957  * into the flow.
 958  *
 959  * @param[in] dev
 960  *   Pointer to the Ethernet device structure.
 961  * @param[in] action
 962  *   Action configuration.
 963  * @param[in] dev_flow
 964  *   Pointer to mlx5_flow.
 965  * @param[out] error
 966  *   Pointer to error structure.
 967  *
 968  * @return
 969  *   0 On success else a negative errno value is returned and rte_errno is set.
 970  */
 971 static int
 972 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
 973                                   const struct rte_flow_action *action,
 974                                   struct rte_eth_dev *dev,
 975                                   struct rte_flow_error *error)
 976 {
 977         const struct rte_flow_action_count *count = action->conf;
 978         struct rte_flow *flow = dev_flow->flow;
 979 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 980         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 981         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 982         struct ibv_flow_spec_counter_action counter = {
 983                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
 984                 .size = size,
 985         };
 986 #endif
 987
 988         if (!flow->counter) {
 989                 flow->counter = flow_verbs_counter_new(dev, count->shared,
 990                                                        count->id);
 991                 if (!flow->counter)
 992                         return rte_flow_error_set(error, rte_errno,
 993                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 994                                                   action,
 995                                                   "cannot get counter"
 996                                                   " context.");
 997         }
 998 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 999         counter.counter_set_handle = flow->counter->cs->handle;
1000         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1001 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1002         counter.counters = flow->counter->cs;
1003         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1004 #endif
1005         return 0;
1006 }
1007
1008 /**
1009  * Internal validation function. For validating both actions and items.
1010  *
1011  * @param[in] dev
1012  *   Pointer to the Ethernet device structure.
1013  * @param[in] attr
1014  *   Pointer to the flow attributes.
1015  * @param[in] items
1016  *   Pointer to the list of items.
1017  * @param[in] actions
1018  *   Pointer to the list of actions.
1019  * @param[in] external
1020  *   This flow rule is created by request external to PMD.
1021  * @param[out] error
1022  *   Pointer to the error structure.
1023  *
1024  * @return
1025  *   0 on success, a negative errno value otherwise and rte_errno is set.
1026  */
1027 static int
1028 flow_verbs_validate(struct rte_eth_dev *dev,
1029                     const struct rte_flow_attr *attr,
1030                     const struct rte_flow_item items[],
1031                     const struct rte_flow_action actions[],
1032                     bool external __rte_unused,
1033                     struct rte_flow_error *error)
1034 {
1035         int ret;
1036         uint64_t action_flags = 0;
1037         uint64_t item_flags = 0;
1038         uint64_t last_item = 0;
1039         uint8_t next_protocol = 0xff;
1040         uint16_t ether_type = 0;
1041
1042         if (items == NULL)
1043                 return -1;
1044         ret = mlx5_flow_validate_attributes(dev, attr, error);
1045         if (ret < 0)
1046                 return ret;
1047         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1048                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1049                 int ret = 0;
1050
1051                 switch (items->type) {
1052                 case RTE_FLOW_ITEM_TYPE_VOID:
1053                         break;
1054                 case RTE_FLOW_ITEM_TYPE_ETH:
1055                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1056                                                           error);
1057                         if (ret < 0)
1058                                 return ret;
1059                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1060                                              MLX5_FLOW_LAYER_OUTER_L2;
1061                         if (items->mask != NULL && items->spec != NULL) {
1062                                 ether_type =
1063                                         ((const struct rte_flow_item_eth *)
1064                                          items->spec)->type;
1065                                 ether_type &=
1066                                         ((const struct rte_flow_item_eth *)
1067                                          items->mask)->type;
1068                                 ether_type = rte_be_to_cpu_16(ether_type);
1069                         } else {
1070                                 ether_type = 0;
1071                         }
1072                         break;
1073                 case RTE_FLOW_ITEM_TYPE_VLAN:
1074                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1075                                                            dev, error);
1076                         if (ret < 0)
1077                                 return ret;
1078                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1079                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1080                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1081                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1082                         if (items->mask != NULL && items->spec != NULL) {
1083                                 ether_type =
1084                                         ((const struct rte_flow_item_vlan *)
1085                                          items->spec)->inner_type;
1086                                 ether_type &=
1087                                         ((const struct rte_flow_item_vlan *)
1088                                          items->mask)->inner_type;
1089                                 ether_type = rte_be_to_cpu_16(ether_type);
1090                         } else {
1091                                 ether_type = 0;
1092                         }
1093                         break;
1094                 case RTE_FLOW_ITEM_TYPE_IPV4:
1095                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1096                                                            last_item,
1097                                                            ether_type, NULL,
1098                                                            error);
1099                         if (ret < 0)
1100                                 return ret;
1101                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1102                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1103                         if (items->mask != NULL &&
1104                             ((const struct rte_flow_item_ipv4 *)
1105                              items->mask)->hdr.next_proto_id) {
1106                                 next_protocol =
1107                                         ((const struct rte_flow_item_ipv4 *)
1108                                          (items->spec))->hdr.next_proto_id;
1109                                 next_protocol &=
1110                                         ((const struct rte_flow_item_ipv4 *)
1111                                          (items->mask))->hdr.next_proto_id;
1112                         } else {
1113                                 /* Reset for inner layer. */
1114                                 next_protocol = 0xff;
1115                         }
1116                         break;
1117                 case RTE_FLOW_ITEM_TYPE_IPV6:
1118                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1119                                                            last_item,
1120                                                            ether_type, NULL,
1121                                                            error);
1122                         if (ret < 0)
1123                                 return ret;
1124                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1125                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1126                         if (items->mask != NULL &&
1127                             ((const struct rte_flow_item_ipv6 *)
1128                              items->mask)->hdr.proto) {
1129                                 next_protocol =
1130                                         ((const struct rte_flow_item_ipv6 *)
1131                                          items->spec)->hdr.proto;
1132                                 next_protocol &=
1133                                         ((const struct rte_flow_item_ipv6 *)
1134                                          items->mask)->hdr.proto;
1135                         } else {
1136                                 /* Reset for inner layer. */
1137                                 next_protocol = 0xff;
1138                         }
1139                         break;
1140                 case RTE_FLOW_ITEM_TYPE_UDP:
1141                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1142                                                           next_protocol,
1143                                                           error);
1144                         if (ret < 0)
1145                                 return ret;
1146                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1147                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1148                         break;
1149                 case RTE_FLOW_ITEM_TYPE_TCP:
1150                         ret = mlx5_flow_validate_item_tcp
1151                                                 (items, item_flags,
1152                                                  next_protocol,
1153                                                  &rte_flow_item_tcp_mask,
1154                                                  error);
1155                         if (ret < 0)
1156                                 return ret;
1157                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1158                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1159                         break;
1160                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1161                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1162                                                             error);
1163                         if (ret < 0)
1164                                 return ret;
1165                         last_item = MLX5_FLOW_LAYER_VXLAN;
1166                         break;
1167                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1168                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1169                                                                 item_flags,
1170                                                                 dev, error);
1171                         if (ret < 0)
1172                                 return ret;
1173                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1174                         break;
1175                 case RTE_FLOW_ITEM_TYPE_GRE:
1176                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1177                                                           next_protocol, error);
1178                         if (ret < 0)
1179                                 return ret;
1180                         last_item = MLX5_FLOW_LAYER_GRE;
1181                         break;
1182                 case RTE_FLOW_ITEM_TYPE_MPLS:
1183                         ret = mlx5_flow_validate_item_mpls(dev, items,
1184                                                            item_flags,
1185                                                            last_item, error);
1186                         if (ret < 0)
1187                                 return ret;
1188                         last_item = MLX5_FLOW_LAYER_MPLS;
1189                         break;
1190                 default:
1191                         return rte_flow_error_set(error, ENOTSUP,
1192                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1193                                                   NULL, "item not supported");
1194                 }
1195                 item_flags |= last_item;
1196         }
1197         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1198                 switch (actions->type) {
1199                 case RTE_FLOW_ACTION_TYPE_VOID:
1200                         break;
1201                 case RTE_FLOW_ACTION_TYPE_FLAG:
1202                         ret = mlx5_flow_validate_action_flag(action_flags,
1203                                                              attr,
1204                                                              error);
1205                         if (ret < 0)
1206                                 return ret;
1207                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1208                         break;
1209                 case RTE_FLOW_ACTION_TYPE_MARK:
1210                         ret = mlx5_flow_validate_action_mark(actions,
1211                                                              action_flags,
1212                                                              attr,
1213                                                              error);
1214                         if (ret < 0)
1215                                 return ret;
1216                         action_flags |= MLX5_FLOW_ACTION_MARK;
1217                         break;
1218                 case RTE_FLOW_ACTION_TYPE_DROP:
1219                         ret = mlx5_flow_validate_action_drop(action_flags,
1220                                                              attr,
1221                                                              error);
1222                         if (ret < 0)
1223                                 return ret;
1224                         action_flags |= MLX5_FLOW_ACTION_DROP;
1225                         break;
1226                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1227                         ret = mlx5_flow_validate_action_queue(actions,
1228                                                               action_flags, dev,
1229                                                               attr,
1230                                                               error);
1231                         if (ret < 0)
1232                                 return ret;
1233                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1234                         break;
1235                 case RTE_FLOW_ACTION_TYPE_RSS:
1236                         ret = mlx5_flow_validate_action_rss(actions,
1237                                                             action_flags, dev,
1238                                                             attr, item_flags,
1239                                                             error);
1240                         if (ret < 0)
1241                                 return ret;
1242                         action_flags |= MLX5_FLOW_ACTION_RSS;
1243                         break;
1244                 case RTE_FLOW_ACTION_TYPE_COUNT:
1245                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1246                         if (ret < 0)
1247                                 return ret;
1248                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1249                         break;
1250                 default:
1251                         return rte_flow_error_set(error, ENOTSUP,
1252                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1253                                                   actions,
1254                                                   "action not supported");
1255                 }
1256         }
1257         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1258                 return rte_flow_error_set(error, EINVAL,
1259                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1260                                           "no fate action is found");
1261         return 0;
1262 }
1263
1264 /**
1265  * Calculate the required bytes that are needed for the action part of the verbs
1266  * flow.
1267  *
1268  * @param[in] actions
1269  *   Pointer to the list of actions.
1270  *
1271  * @return
1272  *   The size of the memory needed for all actions.
1273  */
1274 static int
1275 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1276 {
1277         int size = 0;
1278
1279         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1280                 switch (actions->type) {
1281                 case RTE_FLOW_ACTION_TYPE_VOID:
1282                         break;
1283                 case RTE_FLOW_ACTION_TYPE_FLAG:
1284                         size += sizeof(struct ibv_flow_spec_action_tag);
1285                         break;
1286                 case RTE_FLOW_ACTION_TYPE_MARK:
1287                         size += sizeof(struct ibv_flow_spec_action_tag);
1288                         break;
1289                 case RTE_FLOW_ACTION_TYPE_DROP:
1290                         size += sizeof(struct ibv_flow_spec_action_drop);
1291                         break;
1292                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1293                         break;
1294                 case RTE_FLOW_ACTION_TYPE_RSS:
1295                         break;
1296                 case RTE_FLOW_ACTION_TYPE_COUNT:
1297 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1298         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1299                         size += sizeof(struct ibv_flow_spec_counter_action);
1300 #endif
1301                         break;
1302                 default:
1303                         break;
1304                 }
1305         }
1306         return size;
1307 }
1308
1309 /**
1310  * Calculate the required bytes that are needed for the item part of the verbs
1311  * flow.
1312  *
1313  * @param[in] items
1314  *   Pointer to the list of items.
1315  *
1316  * @return
1317  *   The size of the memory needed for all items.
1318  */
1319 static int
1320 flow_verbs_get_items_size(const struct rte_flow_item items[])
1321 {
1322         int size = 0;
1323
1324         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1325                 switch (items->type) {
1326                 case RTE_FLOW_ITEM_TYPE_VOID:
1327                         break;
1328                 case RTE_FLOW_ITEM_TYPE_ETH:
1329                         size += sizeof(struct ibv_flow_spec_eth);
1330                         break;
1331                 case RTE_FLOW_ITEM_TYPE_VLAN:
1332                         size += sizeof(struct ibv_flow_spec_eth);
1333                         break;
1334                 case RTE_FLOW_ITEM_TYPE_IPV4:
1335                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1336                         break;
1337                 case RTE_FLOW_ITEM_TYPE_IPV6:
1338                         size += sizeof(struct ibv_flow_spec_ipv6);
1339                         break;
1340                 case RTE_FLOW_ITEM_TYPE_UDP:
1341                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1342                         break;
1343                 case RTE_FLOW_ITEM_TYPE_TCP:
1344                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1345                         break;
1346                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1347                         size += sizeof(struct ibv_flow_spec_tunnel);
1348                         break;
1349                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1350                         size += sizeof(struct ibv_flow_spec_tunnel);
1351                         break;
1352 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1353                 case RTE_FLOW_ITEM_TYPE_GRE:
1354                         size += sizeof(struct ibv_flow_spec_gre);
1355                         break;
1356                 case RTE_FLOW_ITEM_TYPE_MPLS:
1357                         size += sizeof(struct ibv_flow_spec_mpls);
1358                         break;
1359 #else
1360                 case RTE_FLOW_ITEM_TYPE_GRE:
1361                         size += sizeof(struct ibv_flow_spec_tunnel);
1362                         break;
1363 #endif
1364                 default:
1365                         break;
1366                 }
1367         }
1368         return size;
1369 }
1370
1371 /**
1372  * Internal preparation function. Allocate mlx5_flow with the required size.
1373  * The required size is calculate based on the actions and items. This function
1374  * also returns the detected actions and items for later use.
1375  *
1376  * @param[in] attr
1377  *   Pointer to the flow attributes.
1378  * @param[in] items
1379  *   Pointer to the list of items.
1380  * @param[in] actions
1381  *   Pointer to the list of actions.
1382  * @param[out] error
1383  *   Pointer to the error structure.
1384  *
1385  * @return
1386  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1387  *   is set.
1388  */
1389 static struct mlx5_flow *
1390 flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
1391                    const struct rte_flow_item items[],
1392                    const struct rte_flow_action actions[],
1393                    struct rte_flow_error *error)
1394 {
1395         uint32_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
1396         struct mlx5_flow *flow;
1397
1398         size += flow_verbs_get_actions_size(actions);
1399         size += flow_verbs_get_items_size(items);
1400         flow = rte_calloc(__func__, 1, size, 0);
1401         if (!flow) {
1402                 rte_flow_error_set(error, ENOMEM,
1403                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1404                                    "not enough memory to create flow");
1405                 return NULL;
1406         }
1407         flow->verbs.attr = (void *)(flow + 1);
1408         flow->verbs.specs =
1409                 (uint8_t *)(flow + 1) + sizeof(struct ibv_flow_attr);
1410         return flow;
1411 }
1412
1413 /**
1414  * Fill the flow with verb spec.
1415  *
1416  * @param[in] dev
1417  *   Pointer to Ethernet device.
1418  * @param[in, out] dev_flow
1419  *   Pointer to the mlx5 flow.
1420  * @param[in] attr
1421  *   Pointer to the flow attributes.
1422  * @param[in] items
1423  *   Pointer to the list of items.
1424  * @param[in] actions
1425  *   Pointer to the list of actions.
1426  * @param[out] error
1427  *   Pointer to the error structure.
1428  *
1429  * @return
1430  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1431  */
1432 static int
1433 flow_verbs_translate(struct rte_eth_dev *dev,
1434                      struct mlx5_flow *dev_flow,
1435                      const struct rte_flow_attr *attr,
1436                      const struct rte_flow_item items[],
1437                      const struct rte_flow_action actions[],
1438                      struct rte_flow_error *error)
1439 {
1440         uint64_t item_flags = 0;
1441         uint64_t action_flags = 0;
1442         uint64_t priority = attr->priority;
1443         uint32_t subpriority = 0;
1444         struct mlx5_priv *priv = dev->data->dev_private;
1445
1446         if (priority == MLX5_FLOW_PRIO_RSVD)
1447                 priority = priv->config.flow_prio - 1;
1448         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1449                 int ret;
1450
1451                 switch (actions->type) {
1452                 case RTE_FLOW_ACTION_TYPE_VOID:
1453                         break;
1454                 case RTE_FLOW_ACTION_TYPE_FLAG:
1455                         flow_verbs_translate_action_flag(dev_flow, actions);
1456                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1457                         break;
1458                 case RTE_FLOW_ACTION_TYPE_MARK:
1459                         flow_verbs_translate_action_mark(dev_flow, actions);
1460                         action_flags |= MLX5_FLOW_ACTION_MARK;
1461                         break;
1462                 case RTE_FLOW_ACTION_TYPE_DROP:
1463                         flow_verbs_translate_action_drop(dev_flow, actions);
1464                         action_flags |= MLX5_FLOW_ACTION_DROP;
1465                         break;
1466                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1467                         flow_verbs_translate_action_queue(dev_flow, actions);
1468                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1469                         break;
1470                 case RTE_FLOW_ACTION_TYPE_RSS:
1471                         flow_verbs_translate_action_rss(dev_flow, actions);
1472                         action_flags |= MLX5_FLOW_ACTION_RSS;
1473                         break;
1474                 case RTE_FLOW_ACTION_TYPE_COUNT:
1475                         ret = flow_verbs_translate_action_count(dev_flow,
1476                                                                 actions,
1477                                                                 dev, error);
1478                         if (ret < 0)
1479                                 return ret;
1480                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1481                         break;
1482                 default:
1483                         return rte_flow_error_set(error, ENOTSUP,
1484                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1485                                                   actions,
1486                                                   "action not supported");
1487                 }
1488         }
1489         dev_flow->actions = action_flags;
1490         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1491                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1492
1493                 switch (items->type) {
1494                 case RTE_FLOW_ITEM_TYPE_VOID:
1495                         break;
1496                 case RTE_FLOW_ITEM_TYPE_ETH:
1497                         flow_verbs_translate_item_eth(dev_flow, items,
1498                                                       item_flags);
1499                         subpriority = MLX5_PRIORITY_MAP_L2;
1500                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1501                                                MLX5_FLOW_LAYER_OUTER_L2;
1502                         break;
1503                 case RTE_FLOW_ITEM_TYPE_VLAN:
1504                         flow_verbs_translate_item_vlan(dev_flow, items,
1505                                                        item_flags);
1506                         subpriority = MLX5_PRIORITY_MAP_L2;
1507                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1508                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1509                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1510                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1511                         break;
1512                 case RTE_FLOW_ITEM_TYPE_IPV4:
1513                         flow_verbs_translate_item_ipv4(dev_flow, items,
1514                                                        item_flags);
1515                         subpriority = MLX5_PRIORITY_MAP_L3;
1516                         dev_flow->verbs.hash_fields |=
1517                                 mlx5_flow_hashfields_adjust
1518                                         (dev_flow, tunnel,
1519                                          MLX5_IPV4_LAYER_TYPES,
1520                                          MLX5_IPV4_IBV_RX_HASH);
1521                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1522                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1523                         break;
1524                 case RTE_FLOW_ITEM_TYPE_IPV6:
1525                         flow_verbs_translate_item_ipv6(dev_flow, items,
1526                                                        item_flags);
1527                         subpriority = MLX5_PRIORITY_MAP_L3;
1528                         dev_flow->verbs.hash_fields |=
1529                                 mlx5_flow_hashfields_adjust
1530                                         (dev_flow, tunnel,
1531                                          MLX5_IPV6_LAYER_TYPES,
1532                                          MLX5_IPV6_IBV_RX_HASH);
1533                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1534                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1535                         break;
1536                 case RTE_FLOW_ITEM_TYPE_TCP:
1537                         flow_verbs_translate_item_tcp(dev_flow, items,
1538                                                       item_flags);
1539                         subpriority = MLX5_PRIORITY_MAP_L4;
1540                         dev_flow->verbs.hash_fields |=
1541                                 mlx5_flow_hashfields_adjust
1542                                         (dev_flow, tunnel, ETH_RSS_TCP,
1543                                          (IBV_RX_HASH_SRC_PORT_TCP |
1544                                           IBV_RX_HASH_DST_PORT_TCP));
1545                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1546                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1547                         break;
1548                 case RTE_FLOW_ITEM_TYPE_UDP:
1549                         flow_verbs_translate_item_udp(dev_flow, items,
1550                                                       item_flags);
1551                         subpriority = MLX5_PRIORITY_MAP_L4;
1552                         dev_flow->verbs.hash_fields |=
1553                                 mlx5_flow_hashfields_adjust
1554                                         (dev_flow, tunnel, ETH_RSS_UDP,
1555                                          (IBV_RX_HASH_SRC_PORT_UDP |
1556                                           IBV_RX_HASH_DST_PORT_UDP));
1557                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1558                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1559                         break;
1560                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1561                         flow_verbs_translate_item_vxlan(dev_flow, items,
1562                                                         item_flags);
1563                         subpriority = MLX5_PRIORITY_MAP_L2;
1564                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1565                         break;
1566                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1567                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1568                                                             item_flags);
1569                         subpriority = MLX5_PRIORITY_MAP_L2;
1570                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1571                         break;
1572                 case RTE_FLOW_ITEM_TYPE_GRE:
1573                         flow_verbs_translate_item_gre(dev_flow, items,
1574                                                       item_flags);
1575                         subpriority = MLX5_PRIORITY_MAP_L2;
1576                         item_flags |= MLX5_FLOW_LAYER_GRE;
1577                         break;
1578                 case RTE_FLOW_ITEM_TYPE_MPLS:
1579                         flow_verbs_translate_item_mpls(dev_flow, items,
1580                                                        item_flags);
1581                         subpriority = MLX5_PRIORITY_MAP_L2;
1582                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1583                         break;
1584                 default:
1585                         return rte_flow_error_set(error, ENOTSUP,
1586                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1587                                                   NULL,
1588                                                   "item not supported");
1589                 }
1590         }
1591         dev_flow->layers = item_flags;
1592         dev_flow->verbs.attr->priority =
1593                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1594         dev_flow->verbs.attr->port = (uint8_t)priv->ibv_port;
1595         return 0;
1596 }
1597
1598 /**
1599  * Remove the flow from the NIC but keeps it in memory.
1600  *
1601  * @param[in] dev
1602  *   Pointer to the Ethernet device structure.
1603  * @param[in, out] flow
1604  *   Pointer to flow structure.
1605  */
1606 static void
1607 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1608 {
1609         struct mlx5_flow_verbs *verbs;
1610         struct mlx5_flow *dev_flow;
1611
1612         if (!flow)
1613                 return;
1614         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1615                 verbs = &dev_flow->verbs;
1616                 if (verbs->flow) {
1617                         claim_zero(mlx5_glue->destroy_flow(verbs->flow));
1618                         verbs->flow = NULL;
1619                 }
1620                 if (verbs->hrxq) {
1621                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
1622                                 mlx5_hrxq_drop_release(dev);
1623                         else
1624                                 mlx5_hrxq_release(dev, verbs->hrxq);
1625                         verbs->hrxq = NULL;
1626                 }
1627                 if (dev_flow->verbs.vf_vlan.tag &&
1628                     dev_flow->verbs.vf_vlan.created) {
1629                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1630                 }
1631         }
1632 }
1633
1634 /**
1635  * Remove the flow from the NIC and the memory.
1636  *
1637  * @param[in] dev
1638  *   Pointer to the Ethernet device structure.
1639  * @param[in, out] flow
1640  *   Pointer to flow structure.
1641  */
1642 static void
1643 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1644 {
1645         struct mlx5_flow *dev_flow;
1646
1647         if (!flow)
1648                 return;
1649         flow_verbs_remove(dev, flow);
1650         while (!LIST_EMPTY(&flow->dev_flows)) {
1651                 dev_flow = LIST_FIRST(&flow->dev_flows);
1652                 LIST_REMOVE(dev_flow, next);
1653                 rte_free(dev_flow);
1654         }
1655         if (flow->counter) {
1656                 flow_verbs_counter_release(dev, flow->counter);
1657                 flow->counter = NULL;
1658         }
1659 }
1660
1661 /**
1662  * Apply the flow to the NIC.
1663  *
1664  * @param[in] dev
1665  *   Pointer to the Ethernet device structure.
1666  * @param[in, out] flow
1667  *   Pointer to flow structure.
1668  * @param[out] error
1669  *   Pointer to error structure.
1670  *
1671  * @return
1672  *   0 on success, a negative errno value otherwise and rte_errno is set.
1673  */
1674 static int
1675 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1676                  struct rte_flow_error *error)
1677 {
1678         struct mlx5_priv *priv = dev->data->dev_private;
1679         struct mlx5_flow_verbs *verbs;
1680         struct mlx5_flow *dev_flow;
1681         int err;
1682
1683         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1684                 verbs = &dev_flow->verbs;
1685                 if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) {
1686                         verbs->hrxq = mlx5_hrxq_drop_new(dev);
1687                         if (!verbs->hrxq) {
1688                                 rte_flow_error_set
1689                                         (error, errno,
1690                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1691                                          "cannot get drop hash queue");
1692                                 goto error;
1693                         }
1694                 } else {
1695                         struct mlx5_hrxq *hrxq;
1696
1697                         hrxq = mlx5_hrxq_get(dev, flow->key,
1698                                              MLX5_RSS_HASH_KEY_LEN,
1699                                              verbs->hash_fields,
1700                                              (*flow->queue),
1701                                              flow->rss.queue_num);
1702                         if (!hrxq)
1703                                 hrxq = mlx5_hrxq_new(dev, flow->key,
1704                                                      MLX5_RSS_HASH_KEY_LEN,
1705                                                      verbs->hash_fields,
1706                                                      (*flow->queue),
1707                                                      flow->rss.queue_num,
1708                                                      !!(dev_flow->layers &
1709                                                        MLX5_FLOW_LAYER_TUNNEL));
1710                         if (!hrxq) {
1711                                 rte_flow_error_set
1712                                         (error, rte_errno,
1713                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1714                                          "cannot get hash queue");
1715                                 goto error;
1716                         }
1717                         verbs->hrxq = hrxq;
1718                 }
1719                 verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp,
1720                                                      verbs->attr);
1721                 if (!verbs->flow) {
1722                         rte_flow_error_set(error, errno,
1723                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1724                                            NULL,
1725                                            "hardware refuses to create flow");
1726                         goto error;
1727                 }
1728                 if (priv->vmwa_context &&
1729                     dev_flow->verbs.vf_vlan.tag &&
1730                     !dev_flow->verbs.vf_vlan.created) {
1731                         /*
1732                          * The rule contains the VLAN pattern.
1733                          * For VF we are going to create VLAN
1734                          * interface to make hypervisor set correct
1735                          * e-Switch vport context.
1736                          */
1737                         mlx5_vlan_vmwa_acquire(dev, &dev_flow->verbs.vf_vlan);
1738                 }
1739         }
1740         return 0;
1741 error:
1742         err = rte_errno; /* Save rte_errno before cleanup. */
1743         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1744                 verbs = &dev_flow->verbs;
1745                 if (verbs->hrxq) {
1746                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
1747                                 mlx5_hrxq_drop_release(dev);
1748                         else
1749                                 mlx5_hrxq_release(dev, verbs->hrxq);
1750                         verbs->hrxq = NULL;
1751                 }
1752                 if (dev_flow->verbs.vf_vlan.tag &&
1753                     dev_flow->verbs.vf_vlan.created) {
1754                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1755                 }
1756         }
1757         rte_errno = err; /* Restore rte_errno. */
1758         return -rte_errno;
1759 }
1760
1761 /**
1762  * Query a flow.
1763  *
1764  * @see rte_flow_query()
1765  * @see rte_flow_ops
1766  */
1767 static int
1768 flow_verbs_query(struct rte_eth_dev *dev,
1769                  struct rte_flow *flow,
1770                  const struct rte_flow_action *actions,
1771                  void *data,
1772                  struct rte_flow_error *error)
1773 {
1774         int ret = -EINVAL;
1775
1776         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1777                 switch (actions->type) {
1778                 case RTE_FLOW_ACTION_TYPE_VOID:
1779                         break;
1780                 case RTE_FLOW_ACTION_TYPE_COUNT:
1781                         ret = flow_verbs_counter_query(dev, flow, data, error);
1782                         break;
1783                 default:
1784                         return rte_flow_error_set(error, ENOTSUP,
1785                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1786                                                   actions,
1787                                                   "action not supported");
1788                 }
1789         }
1790         return ret;
1791 }
1792
1793 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1794         .validate = flow_verbs_validate,
1795         .prepare = flow_verbs_prepare,
1796         .translate = flow_verbs_translate,
1797         .apply = flow_verbs_apply,
1798         .remove = flow_verbs_remove,
1799         .destroy = flow_verbs_destroy,
1800         .query = flow_verbs_query,
1801 };