drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28
  29 #include <mlx5_glue.h>
  30 #include <mlx5_prm.h>
  31
  32 #include "mlx5_defs.h"
  33 #include "mlx5.h"
  34 #include "mlx5_flow.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #define VERBS_SPEC_INNER(item_flags) \
  38         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  39
  40 /**
  41  * Create Verbs flow counter with Verbs library.
  42  *
  43  * @param[in] dev
  44  *   Pointer to the Ethernet device structure.
  45  * @param[in, out] counter
  46  *   mlx5 flow counter object, contains the counter id,
  47  *   handle of created Verbs flow counter is returned
  48  *   in cs field (if counters are supported).
  49  *
  50  * @return
  51  *   0 On success else a negative errno value is returned
  52  *   and rte_errno is set.
  53  */
  54 static int
  55 flow_verbs_counter_create(struct rte_eth_dev *dev,
  56                           struct mlx5_flow_counter *counter)
  57 {
  58 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  59         struct mlx5_priv *priv = dev->data->dev_private;
  60         struct ibv_context *ctx = priv->sh->ctx;
  61         struct ibv_counter_set_init_attr init = {
  62                          .counter_set_id = counter->id};
  63
  64         counter->cs = mlx5_glue->create_counter_set(ctx, &init);
  65         if (!counter->cs) {
  66                 rte_errno = ENOTSUP;
  67                 return -ENOTSUP;
  68         }
  69         return 0;
  70 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
  71         struct mlx5_priv *priv = dev->data->dev_private;
  72         struct ibv_context *ctx = priv->sh->ctx;
  73         struct ibv_counters_init_attr init = {0};
  74         struct ibv_counter_attach_attr attach;
  75         int ret;
  76
  77         memset(&attach, 0, sizeof(attach));
  78         counter->cs = mlx5_glue->create_counters(ctx, &init);
  79         if (!counter->cs) {
  80                 rte_errno = ENOTSUP;
  81                 return -ENOTSUP;
  82         }
  83         attach.counter_desc = IBV_COUNTER_PACKETS;
  84         attach.index = 0;
  85         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
  86         if (!ret) {
  87                 attach.counter_desc = IBV_COUNTER_BYTES;
  88                 attach.index = 1;
  89                 ret = mlx5_glue->attach_counters
  90                                         (counter->cs, &attach, NULL);
  91         }
  92         if (ret) {
  93                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
  94                 counter->cs = NULL;
  95                 rte_errno = ret;
  96                 return -ret;
  97         }
  98         return 0;
  99 #else
 100         (void)dev;
 101         (void)counter;
 102         rte_errno = ENOTSUP;
 103         return -ENOTSUP;
 104 #endif
 105 }
 106
 107 /**
 108  * Get a flow counter.
 109  *
 110  * @param[in] dev
 111  *   Pointer to the Ethernet device structure.
 112  * @param[in] shared
 113  *   Indicate if this counter is shared with other flows.
 114  * @param[in] id
 115  *   Counter identifier.
 116  *
 117  * @return
 118  *   A pointer to the counter, NULL otherwise and rte_errno is set.
 119  */
 120 static struct mlx5_flow_counter *
 121 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 122 {
 123         struct mlx5_priv *priv = dev->data->dev_private;
 124         struct mlx5_flow_counter *cnt;
 125         int ret;
 126
 127         if (shared) {
 128                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
 129                         if (cnt->shared && cnt->id == id) {
 130                                 cnt->ref_cnt++;
 131                                 return cnt;
 132                         }
 133                 }
 134         }
 135         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 136         if (!cnt) {
 137                 rte_errno = ENOMEM;
 138                 return NULL;
 139         }
 140         cnt->id = id;
 141         cnt->shared = shared;
 142         cnt->ref_cnt = 1;
 143         cnt->hits = 0;
 144         cnt->bytes = 0;
 145         /* Create counter with Verbs. */
 146         ret = flow_verbs_counter_create(dev, cnt);
 147         if (!ret) {
 148                 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
 149                 return cnt;
 150         }
 151         /* Some error occurred in Verbs library. */
 152         rte_free(cnt);
 153         rte_errno = -ret;
 154         return NULL;
 155 }
 156
 157 /**
 158  * Release a flow counter.
 159  *
 160  * @param[in] dev
 161  *   Pointer to the Ethernet device structure.
 162  * @param[in] counter
 163  *   Pointer to the counter handler.
 164  */
 165 static void
 166 flow_verbs_counter_release(struct rte_eth_dev *dev,
 167                            struct mlx5_flow_counter *counter)
 168 {
 169         struct mlx5_priv *priv = dev->data->dev_private;
 170
 171         if (--counter->ref_cnt == 0) {
 172 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 173                 claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
 174 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 175                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 176 #endif
 177                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
 178                 rte_free(counter);
 179         }
 180 }
 181
 182 /**
 183  * Query a flow counter via Verbs library call.
 184  *
 185  * @see rte_flow_query()
 186  * @see rte_flow_ops
 187  */
 188 static int
 189 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 190                          struct rte_flow *flow, void *data,
 191                          struct rte_flow_error *error)
 192 {
 193 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 194         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 195         if (flow->counter && flow->counter->cs) {
 196                 struct rte_flow_query_count *qc = data;
 197                 uint64_t counters[2] = {0, 0};
 198 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 199                 struct ibv_query_counter_set_attr query_cs_attr = {
 200                         .cs = flow->counter->cs,
 201                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 202                 };
 203                 struct ibv_counter_set_data query_out = {
 204                         .out = counters,
 205                         .outlen = 2 * sizeof(uint64_t),
 206                 };
 207                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 208                                                        &query_out);
 209 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 210                 int err = mlx5_glue->query_counters
 211                                (flow->counter->cs, counters,
 212                                 RTE_DIM(counters),
 213                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 214 #endif
 215                 if (err)
 216                         return rte_flow_error_set
 217                                 (error, err,
 218                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 219                                  NULL,
 220                                  "cannot read counter");
 221                 qc->hits_set = 1;
 222                 qc->bytes_set = 1;
 223                 qc->hits = counters[0] - flow->counter->hits;
 224                 qc->bytes = counters[1] - flow->counter->bytes;
 225                 if (qc->reset) {
 226                         flow->counter->hits = counters[0];
 227                         flow->counter->bytes = counters[1];
 228                 }
 229                 return 0;
 230         }
 231         return rte_flow_error_set(error, EINVAL,
 232                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 233                                   NULL,
 234                                   "flow does not have counter");
 235 #else
 236         (void)flow;
 237         (void)data;
 238         return rte_flow_error_set(error, ENOTSUP,
 239                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 240                                   NULL,
 241                                   "counters are not available");
 242 #endif
 243 }
 244
 245 /**
 246  * Add a verbs item specification into @p verbs.
 247  *
 248  * @param[out] verbs
 249  *   Pointer to verbs structure.
 250  * @param[in] src
 251  *   Create specification.
 252  * @param[in] size
 253  *   Size in bytes of the specification to copy.
 254  */
 255 static void
 256 flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
 257                     void *src, unsigned int size)
 258 {
 259         void *dst;
 260
 261         if (!verbs)
 262                 return;
 263         MLX5_ASSERT(verbs->specs);
 264         dst = (void *)(verbs->specs + verbs->size);
 265         memcpy(dst, src, size);
 266         ++verbs->attr.num_of_specs;
 267         verbs->size += size;
 268 }
 269
 270 /**
 271  * Convert the @p item into a Verbs specification. This function assumes that
 272  * the input is valid and that there is space to insert the requested item
 273  * into the flow.
 274  *
 275  * @param[in, out] dev_flow
 276  *   Pointer to dev_flow structure.
 277  * @param[in] item
 278  *   Item specification.
 279  * @param[in] item_flags
 280  *   Parsed item flags.
 281  */
 282 static void
 283 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 284                               const struct rte_flow_item *item,
 285                               uint64_t item_flags)
 286 {
 287         const struct rte_flow_item_eth *spec = item->spec;
 288         const struct rte_flow_item_eth *mask = item->mask;
 289         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 290         struct ibv_flow_spec_eth eth = {
 291                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 292                 .size = size,
 293         };
 294
 295         if (!mask)
 296                 mask = &rte_flow_item_eth_mask;
 297         if (spec) {
 298                 unsigned int i;
 299
 300                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 301                         RTE_ETHER_ADDR_LEN);
 302                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 303                         RTE_ETHER_ADDR_LEN);
 304                 eth.val.ether_type = spec->type;
 305                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 306                         RTE_ETHER_ADDR_LEN);
 307                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 308                         RTE_ETHER_ADDR_LEN);
 309                 eth.mask.ether_type = mask->type;
 310                 /* Remove unwanted bits from values. */
 311                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 312                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 313                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 314                 }
 315                 eth.val.ether_type &= eth.mask.ether_type;
 316         }
 317         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 318 }
 319
 320 /**
 321  * Update the VLAN tag in the Verbs Ethernet specification.
 322  * This function assumes that the input is valid and there is space to add
 323  * the requested item.
 324  *
 325  * @param[in, out] attr
 326  *   Pointer to Verbs attributes structure.
 327  * @param[in] eth
 328  *   Verbs structure containing the VLAN information to copy.
 329  */
 330 static void
 331 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 332                             struct ibv_flow_spec_eth *eth)
 333 {
 334         unsigned int i;
 335         const enum ibv_flow_spec_type search = eth->type;
 336         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 337                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 338
 339         for (i = 0; i != attr->num_of_specs; ++i) {
 340                 if (hdr->type == search) {
 341                         struct ibv_flow_spec_eth *e =
 342                                 (struct ibv_flow_spec_eth *)hdr;
 343
 344                         e->val.vlan_tag = eth->val.vlan_tag;
 345                         e->mask.vlan_tag = eth->mask.vlan_tag;
 346                         e->val.ether_type = eth->val.ether_type;
 347                         e->mask.ether_type = eth->mask.ether_type;
 348                         break;
 349                 }
 350                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 351         }
 352 }
 353
 354 /**
 355  * Convert the @p item into a Verbs specification. This function assumes that
 356  * the input is valid and that there is space to insert the requested item
 357  * into the flow.
 358  *
 359  * @param[in, out] dev_flow
 360  *   Pointer to dev_flow structure.
 361  * @param[in] item
 362  *   Item specification.
 363  * @param[in] item_flags
 364  *   Parsed item flags.
 365  */
 366 static void
 367 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 368                                const struct rte_flow_item *item,
 369                                uint64_t item_flags)
 370 {
 371         const struct rte_flow_item_vlan *spec = item->spec;
 372         const struct rte_flow_item_vlan *mask = item->mask;
 373         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 374         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 375         struct ibv_flow_spec_eth eth = {
 376                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 377                 .size = size,
 378         };
 379         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 380                                       MLX5_FLOW_LAYER_OUTER_L2;
 381
 382         if (!mask)
 383                 mask = &rte_flow_item_vlan_mask;
 384         if (spec) {
 385                 eth.val.vlan_tag = spec->tci;
 386                 eth.mask.vlan_tag = mask->tci;
 387                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 388                 eth.val.ether_type = spec->inner_type;
 389                 eth.mask.ether_type = mask->inner_type;
 390                 eth.val.ether_type &= eth.mask.ether_type;
 391         }
 392         if (!(item_flags & l2m))
 393                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 394         else
 395                 flow_verbs_item_vlan_update(&dev_flow->verbs.attr, &eth);
 396         if (!tunnel)
 397                 dev_flow->handle->vf_vlan.tag =
 398                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 399 }
 400
 401 /**
 402  * Convert the @p item into a Verbs specification. This function assumes that
 403  * the input is valid and that there is space to insert the requested item
 404  * into the flow.
 405  *
 406  * @param[in, out] dev_flow
 407  *   Pointer to dev_flow structure.
 408  * @param[in] item
 409  *   Item specification.
 410  * @param[in] item_flags
 411  *   Parsed item flags.
 412  */
 413 static void
 414 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 415                                const struct rte_flow_item *item,
 416                                uint64_t item_flags)
 417 {
 418         const struct rte_flow_item_ipv4 *spec = item->spec;
 419         const struct rte_flow_item_ipv4 *mask = item->mask;
 420         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 421         struct ibv_flow_spec_ipv4_ext ipv4 = {
 422                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 423                 .size = size,
 424         };
 425
 426         if (!mask)
 427                 mask = &rte_flow_item_ipv4_mask;
 428         if (spec) {
 429                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 430                         .src_ip = spec->hdr.src_addr,
 431                         .dst_ip = spec->hdr.dst_addr,
 432                         .proto = spec->hdr.next_proto_id,
 433                         .tos = spec->hdr.type_of_service,
 434                 };
 435                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 436                         .src_ip = mask->hdr.src_addr,
 437                         .dst_ip = mask->hdr.dst_addr,
 438                         .proto = mask->hdr.next_proto_id,
 439                         .tos = mask->hdr.type_of_service,
 440                 };
 441                 /* Remove unwanted bits from values. */
 442                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 443                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 444                 ipv4.val.proto &= ipv4.mask.proto;
 445                 ipv4.val.tos &= ipv4.mask.tos;
 446         }
 447         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 448 }
 449
 450 /**
 451  * Convert the @p item into a Verbs specification. This function assumes that
 452  * the input is valid and that there is space to insert the requested item
 453  * into the flow.
 454  *
 455  * @param[in, out] dev_flow
 456  *   Pointer to dev_flow structure.
 457  * @param[in] item
 458  *   Item specification.
 459  * @param[in] item_flags
 460  *   Parsed item flags.
 461  */
 462 static void
 463 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 464                                const struct rte_flow_item *item,
 465                                uint64_t item_flags)
 466 {
 467         const struct rte_flow_item_ipv6 *spec = item->spec;
 468         const struct rte_flow_item_ipv6 *mask = item->mask;
 469         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 470         struct ibv_flow_spec_ipv6 ipv6 = {
 471                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 472                 .size = size,
 473         };
 474
 475         if (!mask)
 476                 mask = &rte_flow_item_ipv6_mask;
 477         if (spec) {
 478                 unsigned int i;
 479                 uint32_t vtc_flow_val;
 480                 uint32_t vtc_flow_mask;
 481
 482                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 483                        RTE_DIM(ipv6.val.src_ip));
 484                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 485                        RTE_DIM(ipv6.val.dst_ip));
 486                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 487                        RTE_DIM(ipv6.mask.src_ip));
 488                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 489                        RTE_DIM(ipv6.mask.dst_ip));
 490                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 491                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 492                 ipv6.val.flow_label =
 493                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 494                                          RTE_IPV6_HDR_FL_SHIFT);
 495                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 496                                          RTE_IPV6_HDR_TC_SHIFT;
 497                 ipv6.val.next_hdr = spec->hdr.proto;
 498                 ipv6.mask.flow_label =
 499                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 500                                          RTE_IPV6_HDR_FL_SHIFT);
 501                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 502                                           RTE_IPV6_HDR_TC_SHIFT;
 503                 ipv6.mask.next_hdr = mask->hdr.proto;
 504                 /* Remove unwanted bits from values. */
 505                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 506                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 507                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 508                 }
 509                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 510                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 511                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 512         }
 513         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 514 }
 515
 516 /**
 517  * Convert the @p item into a Verbs specification. This function assumes that
 518  * the input is valid and that there is space to insert the requested item
 519  * into the flow.
 520  *
 521  * @param[in, out] dev_flow
 522  *   Pointer to dev_flow structure.
 523  * @param[in] item
 524  *   Item specification.
 525  * @param[in] item_flags
 526  *   Parsed item flags.
 527  */
 528 static void
 529 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 530                               const struct rte_flow_item *item,
 531                               uint64_t item_flags __rte_unused)
 532 {
 533         const struct rte_flow_item_tcp *spec = item->spec;
 534         const struct rte_flow_item_tcp *mask = item->mask;
 535         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 536         struct ibv_flow_spec_tcp_udp tcp = {
 537                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 538                 .size = size,
 539         };
 540
 541         if (!mask)
 542                 mask = &rte_flow_item_tcp_mask;
 543         if (spec) {
 544                 tcp.val.dst_port = spec->hdr.dst_port;
 545                 tcp.val.src_port = spec->hdr.src_port;
 546                 tcp.mask.dst_port = mask->hdr.dst_port;
 547                 tcp.mask.src_port = mask->hdr.src_port;
 548                 /* Remove unwanted bits from values. */
 549                 tcp.val.src_port &= tcp.mask.src_port;
 550                 tcp.val.dst_port &= tcp.mask.dst_port;
 551         }
 552         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 553 }
 554
 555 /**
 556  * Convert the @p item into a Verbs specification. This function assumes that
 557  * the input is valid and that there is space to insert the requested item
 558  * into the flow.
 559  *
 560  * @param[in, out] dev_flow
 561  *   Pointer to dev_flow structure.
 562  * @param[in] item
 563  *   Item specification.
 564  * @param[in] item_flags
 565  *   Parsed item flags.
 566  */
 567 static void
 568 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 569                               const struct rte_flow_item *item,
 570                               uint64_t item_flags __rte_unused)
 571 {
 572         const struct rte_flow_item_udp *spec = item->spec;
 573         const struct rte_flow_item_udp *mask = item->mask;
 574         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 575         struct ibv_flow_spec_tcp_udp udp = {
 576                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 577                 .size = size,
 578         };
 579
 580         if (!mask)
 581                 mask = &rte_flow_item_udp_mask;
 582         if (spec) {
 583                 udp.val.dst_port = spec->hdr.dst_port;
 584                 udp.val.src_port = spec->hdr.src_port;
 585                 udp.mask.dst_port = mask->hdr.dst_port;
 586                 udp.mask.src_port = mask->hdr.src_port;
 587                 /* Remove unwanted bits from values. */
 588                 udp.val.src_port &= udp.mask.src_port;
 589                 udp.val.dst_port &= udp.mask.dst_port;
 590         }
 591         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 592 }
 593
 594 /**
 595  * Convert the @p item into a Verbs specification. This function assumes that
 596  * the input is valid and that there is space to insert the requested item
 597  * into the flow.
 598  *
 599  * @param[in, out] dev_flow
 600  *   Pointer to dev_flow structure.
 601  * @param[in] item
 602  *   Item specification.
 603  * @param[in] item_flags
 604  *   Parsed item flags.
 605  */
 606 static void
 607 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 608                                 const struct rte_flow_item *item,
 609                                 uint64_t item_flags __rte_unused)
 610 {
 611         const struct rte_flow_item_vxlan *spec = item->spec;
 612         const struct rte_flow_item_vxlan *mask = item->mask;
 613         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 614         struct ibv_flow_spec_tunnel vxlan = {
 615                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 616                 .size = size,
 617         };
 618         union vni {
 619                 uint32_t vlan_id;
 620                 uint8_t vni[4];
 621         } id = { .vlan_id = 0, };
 622
 623         if (!mask)
 624                 mask = &rte_flow_item_vxlan_mask;
 625         if (spec) {
 626                 memcpy(&id.vni[1], spec->vni, 3);
 627                 vxlan.val.tunnel_id = id.vlan_id;
 628                 memcpy(&id.vni[1], mask->vni, 3);
 629                 vxlan.mask.tunnel_id = id.vlan_id;
 630                 /* Remove unwanted bits from values. */
 631                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 632         }
 633         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 634 }
 635
 636 /**
 637  * Convert the @p item into a Verbs specification. This function assumes that
 638  * the input is valid and that there is space to insert the requested item
 639  * into the flow.
 640  *
 641  * @param[in, out] dev_flow
 642  *   Pointer to dev_flow structure.
 643  * @param[in] item
 644  *   Item specification.
 645  * @param[in] item_flags
 646  *   Parsed item flags.
 647  */
 648 static void
 649 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 650                                     const struct rte_flow_item *item,
 651                                     uint64_t item_flags __rte_unused)
 652 {
 653         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 654         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 655         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 656         struct ibv_flow_spec_tunnel vxlan_gpe = {
 657                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 658                 .size = size,
 659         };
 660         union vni {
 661                 uint32_t vlan_id;
 662                 uint8_t vni[4];
 663         } id = { .vlan_id = 0, };
 664
 665         if (!mask)
 666                 mask = &rte_flow_item_vxlan_gpe_mask;
 667         if (spec) {
 668                 memcpy(&id.vni[1], spec->vni, 3);
 669                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 670                 memcpy(&id.vni[1], mask->vni, 3);
 671                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 672                 /* Remove unwanted bits from values. */
 673                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 674         }
 675         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 676 }
 677
 678 /**
 679  * Update the protocol in Verbs IPv4/IPv6 spec.
 680  *
 681  * @param[in, out] attr
 682  *   Pointer to Verbs attributes structure.
 683  * @param[in] search
 684  *   Specification type to search in order to update the IP protocol.
 685  * @param[in] protocol
 686  *   Protocol value to set if none is present in the specification.
 687  */
 688 static void
 689 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 690                                        enum ibv_flow_spec_type search,
 691                                        uint8_t protocol)
 692 {
 693         unsigned int i;
 694         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 695                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 696
 697         if (!attr)
 698                 return;
 699         for (i = 0; i != attr->num_of_specs; ++i) {
 700                 if (hdr->type == search) {
 701                         union {
 702                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 703                                 struct ibv_flow_spec_ipv6 *ipv6;
 704                         } ip;
 705
 706                         switch (search) {
 707                         case IBV_FLOW_SPEC_IPV4_EXT:
 708                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 709                                 if (!ip.ipv4->val.proto) {
 710                                         ip.ipv4->val.proto = protocol;
 711                                         ip.ipv4->mask.proto = 0xff;
 712                                 }
 713                                 break;
 714                         case IBV_FLOW_SPEC_IPV6:
 715                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 716                                 if (!ip.ipv6->val.next_hdr) {
 717                                         ip.ipv6->val.next_hdr = protocol;
 718                                         ip.ipv6->mask.next_hdr = 0xff;
 719                                 }
 720                                 break;
 721                         default:
 722                                 break;
 723                         }
 724                         break;
 725                 }
 726                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 727         }
 728 }
 729
 730 /**
 731  * Convert the @p item into a Verbs specification. This function assumes that
 732  * the input is valid and that there is space to insert the requested item
 733  * into the flow.
 734  *
 735  * @param[in, out] dev_flow
 736  *   Pointer to dev_flow structure.
 737  * @param[in] item
 738  *   Item specification.
 739  * @param[in] item_flags
 740  *   Parsed item flags.
 741  */
 742 static void
 743 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 744                               const struct rte_flow_item *item __rte_unused,
 745                               uint64_t item_flags)
 746 {
 747         struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
 748 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 749         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 750         struct ibv_flow_spec_tunnel tunnel = {
 751                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 752                 .size = size,
 753         };
 754 #else
 755         const struct rte_flow_item_gre *spec = item->spec;
 756         const struct rte_flow_item_gre *mask = item->mask;
 757         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 758         struct ibv_flow_spec_gre tunnel = {
 759                 .type = IBV_FLOW_SPEC_GRE,
 760                 .size = size,
 761         };
 762
 763         if (!mask)
 764                 mask = &rte_flow_item_gre_mask;
 765         if (spec) {
 766                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 767                 tunnel.val.protocol = spec->protocol;
 768                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 769                 tunnel.mask.protocol = mask->protocol;
 770                 /* Remove unwanted bits from values. */
 771                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 772                 tunnel.val.protocol &= tunnel.mask.protocol;
 773                 tunnel.val.key &= tunnel.mask.key;
 774         }
 775 #endif
 776         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 777                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 778                                                        IBV_FLOW_SPEC_IPV4_EXT,
 779                                                        IPPROTO_GRE);
 780         else
 781                 flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
 782                                                        IBV_FLOW_SPEC_IPV6,
 783                                                        IPPROTO_GRE);
 784         flow_verbs_spec_add(verbs, &tunnel, size);
 785 }
 786
 787 /**
 788  * Convert the @p action into a Verbs specification. This function assumes that
 789  * the input is valid and that there is space to insert the requested action
 790  * into the flow. This function also return the action that was added.
 791  *
 792  * @param[in, out] dev_flow
 793  *   Pointer to dev_flow structure.
 794  * @param[in] item
 795  *   Item specification.
 796  * @param[in] item_flags
 797  *   Parsed item flags.
 798  */
 799 static void
 800 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
 801                                const struct rte_flow_item *item __rte_unused,
 802                                uint64_t item_flags __rte_unused)
 803 {
 804 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 805         const struct rte_flow_item_mpls *spec = item->spec;
 806         const struct rte_flow_item_mpls *mask = item->mask;
 807         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 808         struct ibv_flow_spec_mpls mpls = {
 809                 .type = IBV_FLOW_SPEC_MPLS,
 810                 .size = size,
 811         };
 812
 813         if (!mask)
 814                 mask = &rte_flow_item_mpls_mask;
 815         if (spec) {
 816                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 817                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 818                 /* Remove unwanted bits from values.  */
 819                 mpls.val.label &= mpls.mask.label;
 820         }
 821         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
 822 #endif
 823 }
 824
 825 /**
 826  * Convert the @p action into a Verbs specification. This function assumes that
 827  * the input is valid and that there is space to insert the requested action
 828  * into the flow.
 829  *
 830  * @param[in] dev_flow
 831  *   Pointer to mlx5_flow.
 832  * @param[in] action
 833  *   Action configuration.
 834  */
 835 static void
 836 flow_verbs_translate_action_drop
 837         (struct mlx5_flow *dev_flow,
 838          const struct rte_flow_action *action __rte_unused)
 839 {
 840         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 841         struct ibv_flow_spec_action_drop drop = {
 842                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 843                         .size = size,
 844         };
 845
 846         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
 847 }
 848
 849 /**
 850  * Convert the @p action into a Verbs specification. This function assumes that
 851  * the input is valid and that there is space to insert the requested action
 852  * into the flow.
 853  *
 854  * @param[in] dev_flow
 855  *   Pointer to mlx5_flow.
 856  * @param[in] action
 857  *   Action configuration.
 858  */
 859 static void
 860 flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow,
 861                                   const struct rte_flow_action *action)
 862 {
 863         const struct rte_flow_action_queue *queue = action->conf;
 864         struct rte_flow *flow = dev_flow->flow;
 865
 866         if (flow->rss.queue)
 867                 (*flow->rss.queue)[0] = queue->index;
 868         flow->rss.queue_num = 1;
 869 }
 870
 871 /**
 872  * Convert the @p action into a Verbs specification. This function assumes that
 873  * the input is valid and that there is space to insert the requested action
 874  * into the flow.
 875  *
 876  * @param[in] action
 877  *   Action configuration.
 878  * @param[in, out] action_flags
 879  *   Pointer to the detected actions.
 880  * @param[in] dev_flow
 881  *   Pointer to mlx5_flow.
 882  */
 883 static void
 884 flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow,
 885                                 const struct rte_flow_action *action)
 886 {
 887         const struct rte_flow_action_rss *rss = action->conf;
 888         const uint8_t *rss_key;
 889         struct rte_flow *flow = dev_flow->flow;
 890
 891         if (flow->rss.queue)
 892                 memcpy((*flow->rss.queue), rss->queue,
 893                        rss->queue_num * sizeof(uint16_t));
 894         flow->rss.queue_num = rss->queue_num;
 895         /* NULL RSS key indicates default RSS key. */
 896         rss_key = !rss->key ? rss_hash_default_key : rss->key;
 897         memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN);
 898         /*
 899          * rss->level and rss.types should be set in advance when expanding
 900          * items for RSS.
 901          */
 902 }
 903
 904 /**
 905  * Convert the @p action into a Verbs specification. This function assumes that
 906  * the input is valid and that there is space to insert the requested action
 907  * into the flow.
 908  *
 909  * @param[in] dev_flow
 910  *   Pointer to mlx5_flow.
 911  * @param[in] action
 912  *   Action configuration.
 913  */
 914 static void
 915 flow_verbs_translate_action_flag
 916         (struct mlx5_flow *dev_flow,
 917          const struct rte_flow_action *action __rte_unused)
 918 {
 919         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 920         struct ibv_flow_spec_action_tag tag = {
 921                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 922                 .size = size,
 923                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 924         };
 925
 926         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 927 }
 928
 929 /**
 930  * Convert the @p action into a Verbs specification. This function assumes that
 931  * the input is valid and that there is space to insert the requested action
 932  * into the flow.
 933  *
 934  * @param[in] dev_flow
 935  *   Pointer to mlx5_flow.
 936  * @param[in] action
 937  *   Action configuration.
 938  */
 939 static void
 940 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
 941                                  const struct rte_flow_action *action)
 942 {
 943         const struct rte_flow_action_mark *mark = action->conf;
 944         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 945         struct ibv_flow_spec_action_tag tag = {
 946                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 947                 .size = size,
 948                 .tag_id = mlx5_flow_mark_set(mark->id),
 949         };
 950
 951         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 952 }
 953
 954 /**
 955  * Convert the @p action into a Verbs specification. This function assumes that
 956  * the input is valid and that there is space to insert the requested action
 957  * into the flow.
 958  *
 959  * @param[in] dev
 960  *   Pointer to the Ethernet device structure.
 961  * @param[in] action
 962  *   Action configuration.
 963  * @param[in] dev_flow
 964  *   Pointer to mlx5_flow.
 965  * @param[out] error
 966  *   Pointer to error structure.
 967  *
 968  * @return
 969  *   0 On success else a negative errno value is returned and rte_errno is set.
 970  */
 971 static int
 972 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
 973                                   const struct rte_flow_action *action,
 974                                   struct rte_eth_dev *dev,
 975                                   struct rte_flow_error *error)
 976 {
 977         const struct rte_flow_action_count *count = action->conf;
 978         struct rte_flow *flow = dev_flow->flow;
 979 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 980         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 981         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 982         struct ibv_flow_spec_counter_action counter = {
 983                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
 984                 .size = size,
 985         };
 986 #endif
 987
 988         if (!flow->counter) {
 989                 flow->counter = flow_verbs_counter_new(dev, count->shared,
 990                                                        count->id);
 991                 if (!flow->counter)
 992                         return rte_flow_error_set(error, rte_errno,
 993                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 994                                                   action,
 995                                                   "cannot get counter"
 996                                                   " context.");
 997         }
 998 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 999         counter.counter_set_handle = flow->counter->cs->handle;
1000         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1001 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1002         counter.counters = flow->counter->cs;
1003         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1004 #endif
1005         return 0;
1006 }
1007
1008 /**
1009  * Internal validation function. For validating both actions and items.
1010  *
1011  * @param[in] dev
1012  *   Pointer to the Ethernet device structure.
1013  * @param[in] attr
1014  *   Pointer to the flow attributes.
1015  * @param[in] items
1016  *   Pointer to the list of items.
1017  * @param[in] actions
1018  *   Pointer to the list of actions.
1019  * @param[in] external
1020  *   This flow rule is created by request external to PMD.
1021  * @param[out] error
1022  *   Pointer to the error structure.
1023  *
1024  * @return
1025  *   0 on success, a negative errno value otherwise and rte_errno is set.
1026  */
1027 static int
1028 flow_verbs_validate(struct rte_eth_dev *dev,
1029                     const struct rte_flow_attr *attr,
1030                     const struct rte_flow_item items[],
1031                     const struct rte_flow_action actions[],
1032                     bool external __rte_unused,
1033                     struct rte_flow_error *error)
1034 {
1035         int ret;
1036         uint64_t action_flags = 0;
1037         uint64_t item_flags = 0;
1038         uint64_t last_item = 0;
1039         uint8_t next_protocol = 0xff;
1040         uint16_t ether_type = 0;
1041
1042         if (items == NULL)
1043                 return -1;
1044         ret = mlx5_flow_validate_attributes(dev, attr, error);
1045         if (ret < 0)
1046                 return ret;
1047         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1048                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1049                 int ret = 0;
1050
1051                 switch (items->type) {
1052                 case RTE_FLOW_ITEM_TYPE_VOID:
1053                         break;
1054                 case RTE_FLOW_ITEM_TYPE_ETH:
1055                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1056                                                           error);
1057                         if (ret < 0)
1058                                 return ret;
1059                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1060                                              MLX5_FLOW_LAYER_OUTER_L2;
1061                         if (items->mask != NULL && items->spec != NULL) {
1062                                 ether_type =
1063                                         ((const struct rte_flow_item_eth *)
1064                                          items->spec)->type;
1065                                 ether_type &=
1066                                         ((const struct rte_flow_item_eth *)
1067                                          items->mask)->type;
1068                                 ether_type = rte_be_to_cpu_16(ether_type);
1069                         } else {
1070                                 ether_type = 0;
1071                         }
1072                         break;
1073                 case RTE_FLOW_ITEM_TYPE_VLAN:
1074                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1075                                                            dev, error);
1076                         if (ret < 0)
1077                                 return ret;
1078                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1079                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1080                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1081                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1082                         if (items->mask != NULL && items->spec != NULL) {
1083                                 ether_type =
1084                                         ((const struct rte_flow_item_vlan *)
1085                                          items->spec)->inner_type;
1086                                 ether_type &=
1087                                         ((const struct rte_flow_item_vlan *)
1088                                          items->mask)->inner_type;
1089                                 ether_type = rte_be_to_cpu_16(ether_type);
1090                         } else {
1091                                 ether_type = 0;
1092                         }
1093                         break;
1094                 case RTE_FLOW_ITEM_TYPE_IPV4:
1095                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1096                                                            last_item,
1097                                                            ether_type, NULL,
1098                                                            error);
1099                         if (ret < 0)
1100                                 return ret;
1101                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1102                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1103                         if (items->mask != NULL &&
1104                             ((const struct rte_flow_item_ipv4 *)
1105                              items->mask)->hdr.next_proto_id) {
1106                                 next_protocol =
1107                                         ((const struct rte_flow_item_ipv4 *)
1108                                          (items->spec))->hdr.next_proto_id;
1109                                 next_protocol &=
1110                                         ((const struct rte_flow_item_ipv4 *)
1111                                          (items->mask))->hdr.next_proto_id;
1112                         } else {
1113                                 /* Reset for inner layer. */
1114                                 next_protocol = 0xff;
1115                         }
1116                         break;
1117                 case RTE_FLOW_ITEM_TYPE_IPV6:
1118                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1119                                                            last_item,
1120                                                            ether_type, NULL,
1121                                                            error);
1122                         if (ret < 0)
1123                                 return ret;
1124                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1125                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1126                         if (items->mask != NULL &&
1127                             ((const struct rte_flow_item_ipv6 *)
1128                              items->mask)->hdr.proto) {
1129                                 next_protocol =
1130                                         ((const struct rte_flow_item_ipv6 *)
1131                                          items->spec)->hdr.proto;
1132                                 next_protocol &=
1133                                         ((const struct rte_flow_item_ipv6 *)
1134                                          items->mask)->hdr.proto;
1135                         } else {
1136                                 /* Reset for inner layer. */
1137                                 next_protocol = 0xff;
1138                         }
1139                         break;
1140                 case RTE_FLOW_ITEM_TYPE_UDP:
1141                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1142                                                           next_protocol,
1143                                                           error);
1144                         if (ret < 0)
1145                                 return ret;
1146                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1147                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1148                         break;
1149                 case RTE_FLOW_ITEM_TYPE_TCP:
1150                         ret = mlx5_flow_validate_item_tcp
1151                                                 (items, item_flags,
1152                                                  next_protocol,
1153                                                  &rte_flow_item_tcp_mask,
1154                                                  error);
1155                         if (ret < 0)
1156                                 return ret;
1157                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1158                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1159                         break;
1160                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1161                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1162                                                             error);
1163                         if (ret < 0)
1164                                 return ret;
1165                         last_item = MLX5_FLOW_LAYER_VXLAN;
1166                         break;
1167                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1168                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1169                                                                 item_flags,
1170                                                                 dev, error);
1171                         if (ret < 0)
1172                                 return ret;
1173                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1174                         break;
1175                 case RTE_FLOW_ITEM_TYPE_GRE:
1176                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1177                                                           next_protocol, error);
1178                         if (ret < 0)
1179                                 return ret;
1180                         last_item = MLX5_FLOW_LAYER_GRE;
1181                         break;
1182                 case RTE_FLOW_ITEM_TYPE_MPLS:
1183                         ret = mlx5_flow_validate_item_mpls(dev, items,
1184                                                            item_flags,
1185                                                            last_item, error);
1186                         if (ret < 0)
1187                                 return ret;
1188                         last_item = MLX5_FLOW_LAYER_MPLS;
1189                         break;
1190                 default:
1191                         return rte_flow_error_set(error, ENOTSUP,
1192                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1193                                                   NULL, "item not supported");
1194                 }
1195                 item_flags |= last_item;
1196         }
1197         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1198                 switch (actions->type) {
1199                 case RTE_FLOW_ACTION_TYPE_VOID:
1200                         break;
1201                 case RTE_FLOW_ACTION_TYPE_FLAG:
1202                         ret = mlx5_flow_validate_action_flag(action_flags,
1203                                                              attr,
1204                                                              error);
1205                         if (ret < 0)
1206                                 return ret;
1207                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1208                         break;
1209                 case RTE_FLOW_ACTION_TYPE_MARK:
1210                         ret = mlx5_flow_validate_action_mark(actions,
1211                                                              action_flags,
1212                                                              attr,
1213                                                              error);
1214                         if (ret < 0)
1215                                 return ret;
1216                         action_flags |= MLX5_FLOW_ACTION_MARK;
1217                         break;
1218                 case RTE_FLOW_ACTION_TYPE_DROP:
1219                         ret = mlx5_flow_validate_action_drop(action_flags,
1220                                                              attr,
1221                                                              error);
1222                         if (ret < 0)
1223                                 return ret;
1224                         action_flags |= MLX5_FLOW_ACTION_DROP;
1225                         break;
1226                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1227                         ret = mlx5_flow_validate_action_queue(actions,
1228                                                               action_flags, dev,
1229                                                               attr,
1230                                                               error);
1231                         if (ret < 0)
1232                                 return ret;
1233                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1234                         break;
1235                 case RTE_FLOW_ACTION_TYPE_RSS:
1236                         ret = mlx5_flow_validate_action_rss(actions,
1237                                                             action_flags, dev,
1238                                                             attr, item_flags,
1239                                                             error);
1240                         if (ret < 0)
1241                                 return ret;
1242                         action_flags |= MLX5_FLOW_ACTION_RSS;
1243                         break;
1244                 case RTE_FLOW_ACTION_TYPE_COUNT:
1245                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1246                         if (ret < 0)
1247                                 return ret;
1248                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1249                         break;
1250                 default:
1251                         return rte_flow_error_set(error, ENOTSUP,
1252                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1253                                                   actions,
1254                                                   "action not supported");
1255                 }
1256         }
1257         /*
1258          * Validate the drop action mutual exclusion with other actions.
1259          * Drop action is mutually-exclusive with any other action, except for
1260          * Count action.
1261          */
1262         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1263             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1264                 return rte_flow_error_set(error, EINVAL,
1265                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1266                                           "Drop action is mutually-exclusive "
1267                                           "with any other action, except for "
1268                                           "Count action");
1269         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1270                 return rte_flow_error_set(error, EINVAL,
1271                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1272                                           "no fate action is found");
1273         return 0;
1274 }
1275
1276 /**
1277  * Calculate the required bytes that are needed for the action part of the verbs
1278  * flow.
1279  *
1280  * @param[in] actions
1281  *   Pointer to the list of actions.
1282  *
1283  * @return
1284  *   The size of the memory needed for all actions.
1285  */
1286 static int
1287 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1288 {
1289         int size = 0;
1290
1291         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1292                 switch (actions->type) {
1293                 case RTE_FLOW_ACTION_TYPE_VOID:
1294                         break;
1295                 case RTE_FLOW_ACTION_TYPE_FLAG:
1296                         size += sizeof(struct ibv_flow_spec_action_tag);
1297                         break;
1298                 case RTE_FLOW_ACTION_TYPE_MARK:
1299                         size += sizeof(struct ibv_flow_spec_action_tag);
1300                         break;
1301                 case RTE_FLOW_ACTION_TYPE_DROP:
1302                         size += sizeof(struct ibv_flow_spec_action_drop);
1303                         break;
1304                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1305                         break;
1306                 case RTE_FLOW_ACTION_TYPE_RSS:
1307                         break;
1308                 case RTE_FLOW_ACTION_TYPE_COUNT:
1309 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1310         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1311                         size += sizeof(struct ibv_flow_spec_counter_action);
1312 #endif
1313                         break;
1314                 default:
1315                         break;
1316                 }
1317         }
1318         return size;
1319 }
1320
1321 /**
1322  * Calculate the required bytes that are needed for the item part of the verbs
1323  * flow.
1324  *
1325  * @param[in] items
1326  *   Pointer to the list of items.
1327  *
1328  * @return
1329  *   The size of the memory needed for all items.
1330  */
1331 static int
1332 flow_verbs_get_items_size(const struct rte_flow_item items[])
1333 {
1334         int size = 0;
1335
1336         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1337                 switch (items->type) {
1338                 case RTE_FLOW_ITEM_TYPE_VOID:
1339                         break;
1340                 case RTE_FLOW_ITEM_TYPE_ETH:
1341                         size += sizeof(struct ibv_flow_spec_eth);
1342                         break;
1343                 case RTE_FLOW_ITEM_TYPE_VLAN:
1344                         size += sizeof(struct ibv_flow_spec_eth);
1345                         break;
1346                 case RTE_FLOW_ITEM_TYPE_IPV4:
1347                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1348                         break;
1349                 case RTE_FLOW_ITEM_TYPE_IPV6:
1350                         size += sizeof(struct ibv_flow_spec_ipv6);
1351                         break;
1352                 case RTE_FLOW_ITEM_TYPE_UDP:
1353                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1354                         break;
1355                 case RTE_FLOW_ITEM_TYPE_TCP:
1356                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1357                         break;
1358                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1359                         size += sizeof(struct ibv_flow_spec_tunnel);
1360                         break;
1361                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1362                         size += sizeof(struct ibv_flow_spec_tunnel);
1363                         break;
1364 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1365                 case RTE_FLOW_ITEM_TYPE_GRE:
1366                         size += sizeof(struct ibv_flow_spec_gre);
1367                         break;
1368                 case RTE_FLOW_ITEM_TYPE_MPLS:
1369                         size += sizeof(struct ibv_flow_spec_mpls);
1370                         break;
1371 #else
1372                 case RTE_FLOW_ITEM_TYPE_GRE:
1373                         size += sizeof(struct ibv_flow_spec_tunnel);
1374                         break;
1375 #endif
1376                 default:
1377                         break;
1378                 }
1379         }
1380         return size;
1381 }
1382
1383 /**
1384  * Internal preparation function. Allocate mlx5_flow with the required size.
1385  * The required size is calculate based on the actions and items. This function
1386  * also returns the detected actions and items for later use.
1387  *
1388  * @param[in] dev
1389  *   Pointer to Ethernet device.
1390  * @param[in] attr
1391  *   Pointer to the flow attributes.
1392  * @param[in] items
1393  *   Pointer to the list of items.
1394  * @param[in] actions
1395  *   Pointer to the list of actions.
1396  * @param[out] error
1397  *   Pointer to the error structure.
1398  *
1399  * @return
1400  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1401  *   is set.
1402  */
1403 static struct mlx5_flow *
1404 flow_verbs_prepare(struct rte_eth_dev *dev,
1405                    const struct rte_flow_attr *attr __rte_unused,
1406                    const struct rte_flow_item items[],
1407                    const struct rte_flow_action actions[],
1408                    struct rte_flow_error *error)
1409 {
1410         size_t size = 0;
1411         struct mlx5_flow *dev_flow;
1412         struct mlx5_flow_handle *dev_handle;
1413         struct mlx5_priv *priv = dev->data->dev_private;
1414
1415         size += flow_verbs_get_actions_size(actions);
1416         size += flow_verbs_get_items_size(items);
1417         if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
1418                 rte_flow_error_set(error, E2BIG,
1419                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1420                                    "Verbs spec/action size too large");
1421                 return NULL;
1422         }
1423         /* In case of corrupting the memory. */
1424         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
1425                 rte_flow_error_set(error, ENOSPC,
1426                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1427                                    "not free temporary device flow");
1428                 return NULL;
1429         }
1430         dev_handle = rte_calloc(__func__, 1, MLX5_FLOW_HANDLE_VERBS_SIZE, 0);
1431         if (!dev_handle) {
1432                 rte_flow_error_set(error, ENOMEM,
1433                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1434                                    "not enough memory to create flow handle");
1435                 return NULL;
1436         }
1437         /* No multi-thread supporting. */
1438         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
1439         dev_flow->handle = dev_handle;
1440         /* Memcpy is used, only size needs to be cleared to 0. */
1441         dev_flow->verbs.size = 0;
1442         dev_flow->verbs.attr.num_of_specs = 0;
1443         dev_flow->ingress = attr->ingress;
1444         /* Need to set transfer attribute: not supported in Verbs mode. */
1445         return dev_flow;
1446 }
1447
1448 /**
1449  * Fill the flow with verb spec.
1450  *
1451  * @param[in] dev
1452  *   Pointer to Ethernet device.
1453  * @param[in, out] dev_flow
1454  *   Pointer to the mlx5 flow.
1455  * @param[in] attr
1456  *   Pointer to the flow attributes.
1457  * @param[in] items
1458  *   Pointer to the list of items.
1459  * @param[in] actions
1460  *   Pointer to the list of actions.
1461  * @param[out] error
1462  *   Pointer to the error structure.
1463  *
1464  * @return
1465  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1466  */
1467 static int
1468 flow_verbs_translate(struct rte_eth_dev *dev,
1469                      struct mlx5_flow *dev_flow,
1470                      const struct rte_flow_attr *attr,
1471                      const struct rte_flow_item items[],
1472                      const struct rte_flow_action actions[],
1473                      struct rte_flow_error *error)
1474 {
1475         uint64_t item_flags = 0;
1476         uint64_t action_flags = 0;
1477         uint64_t priority = attr->priority;
1478         uint32_t subpriority = 0;
1479         struct mlx5_priv *priv = dev->data->dev_private;
1480
1481         if (priority == MLX5_FLOW_PRIO_RSVD)
1482                 priority = priv->config.flow_prio - 1;
1483         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1484                 int ret;
1485
1486                 switch (actions->type) {
1487                 case RTE_FLOW_ACTION_TYPE_VOID:
1488                         break;
1489                 case RTE_FLOW_ACTION_TYPE_FLAG:
1490                         flow_verbs_translate_action_flag(dev_flow, actions);
1491                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1492                         break;
1493                 case RTE_FLOW_ACTION_TYPE_MARK:
1494                         flow_verbs_translate_action_mark(dev_flow, actions);
1495                         action_flags |= MLX5_FLOW_ACTION_MARK;
1496                         break;
1497                 case RTE_FLOW_ACTION_TYPE_DROP:
1498                         flow_verbs_translate_action_drop(dev_flow, actions);
1499                         action_flags |= MLX5_FLOW_ACTION_DROP;
1500                         break;
1501                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1502                         flow_verbs_translate_action_queue(dev_flow, actions);
1503                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1504                         break;
1505                 case RTE_FLOW_ACTION_TYPE_RSS:
1506                         flow_verbs_translate_action_rss(dev_flow, actions);
1507                         action_flags |= MLX5_FLOW_ACTION_RSS;
1508                         break;
1509                 case RTE_FLOW_ACTION_TYPE_COUNT:
1510                         ret = flow_verbs_translate_action_count(dev_flow,
1511                                                                 actions,
1512                                                                 dev, error);
1513                         if (ret < 0)
1514                                 return ret;
1515                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1516                         break;
1517                 default:
1518                         return rte_flow_error_set(error, ENOTSUP,
1519                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1520                                                   actions,
1521                                                   "action not supported");
1522                 }
1523         }
1524         dev_flow->handle->act_flags = action_flags;
1525         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1526                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1527
1528                 switch (items->type) {
1529                 case RTE_FLOW_ITEM_TYPE_VOID:
1530                         break;
1531                 case RTE_FLOW_ITEM_TYPE_ETH:
1532                         flow_verbs_translate_item_eth(dev_flow, items,
1533                                                       item_flags);
1534                         subpriority = MLX5_PRIORITY_MAP_L2;
1535                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1536                                                MLX5_FLOW_LAYER_OUTER_L2;
1537                         break;
1538                 case RTE_FLOW_ITEM_TYPE_VLAN:
1539                         flow_verbs_translate_item_vlan(dev_flow, items,
1540                                                        item_flags);
1541                         subpriority = MLX5_PRIORITY_MAP_L2;
1542                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1543                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1544                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1545                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1546                         break;
1547                 case RTE_FLOW_ITEM_TYPE_IPV4:
1548                         flow_verbs_translate_item_ipv4(dev_flow, items,
1549                                                        item_flags);
1550                         subpriority = MLX5_PRIORITY_MAP_L3;
1551                         dev_flow->hash_fields |=
1552                                 mlx5_flow_hashfields_adjust
1553                                         (dev_flow, tunnel,
1554                                          MLX5_IPV4_LAYER_TYPES,
1555                                          MLX5_IPV4_IBV_RX_HASH);
1556                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1557                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1558                         break;
1559                 case RTE_FLOW_ITEM_TYPE_IPV6:
1560                         flow_verbs_translate_item_ipv6(dev_flow, items,
1561                                                        item_flags);
1562                         subpriority = MLX5_PRIORITY_MAP_L3;
1563                         dev_flow->hash_fields |=
1564                                 mlx5_flow_hashfields_adjust
1565                                         (dev_flow, tunnel,
1566                                          MLX5_IPV6_LAYER_TYPES,
1567                                          MLX5_IPV6_IBV_RX_HASH);
1568                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1569                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1570                         break;
1571                 case RTE_FLOW_ITEM_TYPE_TCP:
1572                         flow_verbs_translate_item_tcp(dev_flow, items,
1573                                                       item_flags);
1574                         subpriority = MLX5_PRIORITY_MAP_L4;
1575                         dev_flow->hash_fields |=
1576                                 mlx5_flow_hashfields_adjust
1577                                         (dev_flow, tunnel, ETH_RSS_TCP,
1578                                          (IBV_RX_HASH_SRC_PORT_TCP |
1579                                           IBV_RX_HASH_DST_PORT_TCP));
1580                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1581                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1582                         break;
1583                 case RTE_FLOW_ITEM_TYPE_UDP:
1584                         flow_verbs_translate_item_udp(dev_flow, items,
1585                                                       item_flags);
1586                         subpriority = MLX5_PRIORITY_MAP_L4;
1587                         dev_flow->hash_fields |=
1588                                 mlx5_flow_hashfields_adjust
1589                                         (dev_flow, tunnel, ETH_RSS_UDP,
1590                                          (IBV_RX_HASH_SRC_PORT_UDP |
1591                                           IBV_RX_HASH_DST_PORT_UDP));
1592                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1593                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1594                         break;
1595                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1596                         flow_verbs_translate_item_vxlan(dev_flow, items,
1597                                                         item_flags);
1598                         subpriority = MLX5_PRIORITY_MAP_L2;
1599                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1600                         break;
1601                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1602                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1603                                                             item_flags);
1604                         subpriority = MLX5_PRIORITY_MAP_L2;
1605                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1606                         break;
1607                 case RTE_FLOW_ITEM_TYPE_GRE:
1608                         flow_verbs_translate_item_gre(dev_flow, items,
1609                                                       item_flags);
1610                         subpriority = MLX5_PRIORITY_MAP_L2;
1611                         item_flags |= MLX5_FLOW_LAYER_GRE;
1612                         break;
1613                 case RTE_FLOW_ITEM_TYPE_MPLS:
1614                         flow_verbs_translate_item_mpls(dev_flow, items,
1615                                                        item_flags);
1616                         subpriority = MLX5_PRIORITY_MAP_L2;
1617                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1618                         break;
1619                 default:
1620                         return rte_flow_error_set(error, ENOTSUP,
1621                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1622                                                   NULL,
1623                                                   "item not supported");
1624                 }
1625         }
1626         dev_flow->handle->layers = item_flags;
1627         /* Other members of attr will be ignored. */
1628         dev_flow->verbs.attr.priority =
1629                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1630         dev_flow->verbs.attr.port = (uint8_t)priv->ibv_port;
1631         return 0;
1632 }
1633
1634 /**
1635  * Remove the flow from the NIC but keeps it in memory.
1636  *
1637  * @param[in] dev
1638  *   Pointer to the Ethernet device structure.
1639  * @param[in, out] flow
1640  *   Pointer to flow structure.
1641  */
1642 static void
1643 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1644 {
1645         struct mlx5_flow_handle *handle;
1646
1647         if (!flow)
1648                 return;
1649         LIST_FOREACH(handle, &flow->dev_handles, next) {
1650                 if (handle->ib_flow) {
1651                         claim_zero(mlx5_glue->destroy_flow(handle->ib_flow));
1652                         handle->ib_flow = NULL;
1653                 }
1654                 if (handle->hrxq) {
1655                         if (handle->act_flags & MLX5_FLOW_ACTION_DROP)
1656                                 mlx5_hrxq_drop_release(dev);
1657                         else
1658                                 mlx5_hrxq_release(dev, handle->hrxq);
1659                         handle->hrxq = NULL;
1660                 }
1661                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1662                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1663         }
1664 }
1665
1666 /**
1667  * Remove the flow from the NIC and the memory.
1668  *
1669  * @param[in] dev
1670  *   Pointer to the Ethernet device structure.
1671  * @param[in, out] flow
1672  *   Pointer to flow structure.
1673  */
1674 static void
1675 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1676 {
1677         struct mlx5_flow_handle *handle;
1678
1679         if (!flow)
1680                 return;
1681         flow_verbs_remove(dev, flow);
1682         while (!LIST_EMPTY(&flow->dev_handles)) {
1683                 handle = LIST_FIRST(&flow->dev_handles);
1684                 LIST_REMOVE(handle, next);
1685                 rte_free(handle);
1686         }
1687         if (flow->counter) {
1688                 flow_verbs_counter_release(dev, flow->counter);
1689                 flow->counter = NULL;
1690         }
1691 }
1692
1693 /**
1694  * Apply the flow to the NIC.
1695  *
1696  * @param[in] dev
1697  *   Pointer to the Ethernet device structure.
1698  * @param[in, out] flow
1699  *   Pointer to flow structure.
1700  * @param[out] error
1701  *   Pointer to error structure.
1702  *
1703  * @return
1704  *   0 on success, a negative errno value otherwise and rte_errno is set.
1705  */
1706 static int
1707 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1708                  struct rte_flow_error *error)
1709 {
1710         struct mlx5_priv *priv = dev->data->dev_private;
1711         struct mlx5_flow_handle *handle;
1712         struct mlx5_flow *dev_flow;
1713         int err;
1714         int idx;
1715
1716         for (idx = priv->flow_idx - 1; idx >= 0; idx--) {
1717                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
1718                 handle = dev_flow->handle;
1719                 if (handle->act_flags & MLX5_FLOW_ACTION_DROP) {
1720                         handle->hrxq = mlx5_hrxq_drop_new(dev);
1721                         if (!handle->hrxq) {
1722                                 rte_flow_error_set
1723                                         (error, errno,
1724                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1725                                          "cannot get drop hash queue");
1726                                 goto error;
1727                         }
1728                 } else {
1729                         struct mlx5_hrxq *hrxq;
1730
1731                         MLX5_ASSERT(flow->rss.queue);
1732                         hrxq = mlx5_hrxq_get(dev, flow->rss.key,
1733                                              MLX5_RSS_HASH_KEY_LEN,
1734                                              dev_flow->hash_fields,
1735                                              (*flow->rss.queue),
1736                                              flow->rss.queue_num);
1737                         if (!hrxq)
1738                                 hrxq = mlx5_hrxq_new(dev, flow->rss.key,
1739                                                 MLX5_RSS_HASH_KEY_LEN,
1740                                                 dev_flow->hash_fields,
1741                                                 (*flow->rss.queue),
1742                                                 flow->rss.queue_num,
1743                                                 !!(handle->layers &
1744                                                 MLX5_FLOW_LAYER_TUNNEL));
1745                         if (!hrxq) {
1746                                 rte_flow_error_set
1747                                         (error, rte_errno,
1748                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1749                                          "cannot get hash queue");
1750                                 goto error;
1751                         }
1752                         handle->hrxq = hrxq;
1753                 }
1754                 handle->ib_flow = mlx5_glue->create_flow(handle->hrxq->qp,
1755                                                      &dev_flow->verbs.attr);
1756                 if (!handle->ib_flow) {
1757                         rte_flow_error_set(error, errno,
1758                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1759                                            NULL,
1760                                            "hardware refuses to create flow");
1761                         goto error;
1762                 }
1763                 if (priv->vmwa_context &&
1764                     handle->vf_vlan.tag && !handle->vf_vlan.created) {
1765                         /*
1766                          * The rule contains the VLAN pattern.
1767                          * For VF we are going to create VLAN
1768                          * interface to make hypervisor set correct
1769                          * e-Switch vport context.
1770                          */
1771                         mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
1772                 }
1773         }
1774         return 0;
1775 error:
1776         err = rte_errno; /* Save rte_errno before cleanup. */
1777         LIST_FOREACH(handle, &flow->dev_handles, next) {
1778                 if (handle->hrxq) {
1779                         if (handle->act_flags & MLX5_FLOW_ACTION_DROP)
1780                                 mlx5_hrxq_drop_release(dev);
1781                         else
1782                                 mlx5_hrxq_release(dev, handle->hrxq);
1783                         handle->hrxq = NULL;
1784                 }
1785                 if (handle->vf_vlan.tag && handle->vf_vlan.created)
1786                         mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
1787         }
1788         rte_errno = err; /* Restore rte_errno. */
1789         return -rte_errno;
1790 }
1791
1792 /**
1793  * Query a flow.
1794  *
1795  * @see rte_flow_query()
1796  * @see rte_flow_ops
1797  */
1798 static int
1799 flow_verbs_query(struct rte_eth_dev *dev,
1800                  struct rte_flow *flow,
1801                  const struct rte_flow_action *actions,
1802                  void *data,
1803                  struct rte_flow_error *error)
1804 {
1805         int ret = -EINVAL;
1806
1807         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1808                 switch (actions->type) {
1809                 case RTE_FLOW_ACTION_TYPE_VOID:
1810                         break;
1811                 case RTE_FLOW_ACTION_TYPE_COUNT:
1812                         ret = flow_verbs_counter_query(dev, flow, data, error);
1813                         break;
1814                 default:
1815                         return rte_flow_error_set(error, ENOTSUP,
1816                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1817                                                   actions,
1818                                                   "action not supported");
1819                 }
1820         }
1821         return ret;
1822 }
1823
1824 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1825         .validate = flow_verbs_validate,
1826         .prepare = flow_verbs_prepare,
1827         .translate = flow_verbs_translate,
1828         .apply = flow_verbs_apply,
1829         .remove = flow_verbs_remove,
1830         .destroy = flow_verbs_destroy,
1831         .query = flow_verbs_query,
1832 };