drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_ip.h>
  28
  29 #include <mlx5_glue.h>
  30 #include <mlx5_prm.h>
  31
  32 #include "mlx5_defs.h"
  33 #include "mlx5.h"
  34 #include "mlx5_flow.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #define VERBS_SPEC_INNER(item_flags) \
  38         (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
  39
  40 /**
  41  * Create Verbs flow counter with Verbs library.
  42  *
  43  * @param[in] dev
  44  *   Pointer to the Ethernet device structure.
  45  * @param[in, out] counter
  46  *   mlx5 flow counter object, contains the counter id,
  47  *   handle of created Verbs flow counter is returned
  48  *   in cs field (if counters are supported).
  49  *
  50  * @return
  51  *   0 On success else a negative errno value is returned
  52  *   and rte_errno is set.
  53  */
  54 static int
  55 flow_verbs_counter_create(struct rte_eth_dev *dev,
  56                           struct mlx5_flow_counter *counter)
  57 {
  58 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  59         struct mlx5_priv *priv = dev->data->dev_private;
  60         struct ibv_context *ctx = priv->sh->ctx;
  61         struct ibv_counter_set_init_attr init = {
  62                          .counter_set_id = counter->id};
  63
  64         counter->cs = mlx5_glue->create_counter_set(ctx, &init);
  65         if (!counter->cs) {
  66                 rte_errno = ENOTSUP;
  67                 return -ENOTSUP;
  68         }
  69         return 0;
  70 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
  71         struct mlx5_priv *priv = dev->data->dev_private;
  72         struct ibv_context *ctx = priv->sh->ctx;
  73         struct ibv_counters_init_attr init = {0};
  74         struct ibv_counter_attach_attr attach;
  75         int ret;
  76
  77         memset(&attach, 0, sizeof(attach));
  78         counter->cs = mlx5_glue->create_counters(ctx, &init);
  79         if (!counter->cs) {
  80                 rte_errno = ENOTSUP;
  81                 return -ENOTSUP;
  82         }
  83         attach.counter_desc = IBV_COUNTER_PACKETS;
  84         attach.index = 0;
  85         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
  86         if (!ret) {
  87                 attach.counter_desc = IBV_COUNTER_BYTES;
  88                 attach.index = 1;
  89                 ret = mlx5_glue->attach_counters
  90                                         (counter->cs, &attach, NULL);
  91         }
  92         if (ret) {
  93                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
  94                 counter->cs = NULL;
  95                 rte_errno = ret;
  96                 return -ret;
  97         }
  98         return 0;
  99 #else
 100         (void)dev;
 101         (void)counter;
 102         rte_errno = ENOTSUP;
 103         return -ENOTSUP;
 104 #endif
 105 }
 106
 107 /**
 108  * Get a flow counter.
 109  *
 110  * @param[in] dev
 111  *   Pointer to the Ethernet device structure.
 112  * @param[in] shared
 113  *   Indicate if this counter is shared with other flows.
 114  * @param[in] id
 115  *   Counter identifier.
 116  *
 117  * @return
 118  *   A pointer to the counter, NULL otherwise and rte_errno is set.
 119  */
 120 static struct mlx5_flow_counter *
 121 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 122 {
 123         struct mlx5_priv *priv = dev->data->dev_private;
 124         struct mlx5_flow_counter *cnt;
 125         int ret;
 126
 127         if (shared) {
 128                 TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
 129                         if (cnt->shared && cnt->id == id) {
 130                                 cnt->ref_cnt++;
 131                                 return cnt;
 132                         }
 133                 }
 134         }
 135         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 136         if (!cnt) {
 137                 rte_errno = ENOMEM;
 138                 return NULL;
 139         }
 140         cnt->id = id;
 141         cnt->shared = shared;
 142         cnt->ref_cnt = 1;
 143         cnt->hits = 0;
 144         cnt->bytes = 0;
 145         /* Create counter with Verbs. */
 146         ret = flow_verbs_counter_create(dev, cnt);
 147         if (!ret) {
 148                 TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
 149                 return cnt;
 150         }
 151         /* Some error occurred in Verbs library. */
 152         rte_free(cnt);
 153         rte_errno = -ret;
 154         return NULL;
 155 }
 156
 157 /**
 158  * Release a flow counter.
 159  *
 160  * @param[in] dev
 161  *   Pointer to the Ethernet device structure.
 162  * @param[in] counter
 163  *   Pointer to the counter handler.
 164  */
 165 static void
 166 flow_verbs_counter_release(struct rte_eth_dev *dev,
 167                            struct mlx5_flow_counter *counter)
 168 {
 169         struct mlx5_priv *priv = dev->data->dev_private;
 170
 171         if (--counter->ref_cnt == 0) {
 172 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 173                 claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
 174 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 175                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 176 #endif
 177                 TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
 178                 rte_free(counter);
 179         }
 180 }
 181
 182 /**
 183  * Query a flow counter via Verbs library call.
 184  *
 185  * @see rte_flow_query()
 186  * @see rte_flow_ops
 187  */
 188 static int
 189 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 190                          struct rte_flow *flow, void *data,
 191                          struct rte_flow_error *error)
 192 {
 193 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 194         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 195         if (flow->counter && flow->counter->cs) {
 196                 struct rte_flow_query_count *qc = data;
 197                 uint64_t counters[2] = {0, 0};
 198 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 199                 struct ibv_query_counter_set_attr query_cs_attr = {
 200                         .cs = flow->counter->cs,
 201                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 202                 };
 203                 struct ibv_counter_set_data query_out = {
 204                         .out = counters,
 205                         .outlen = 2 * sizeof(uint64_t),
 206                 };
 207                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 208                                                        &query_out);
 209 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 210                 int err = mlx5_glue->query_counters
 211                                (flow->counter->cs, counters,
 212                                 RTE_DIM(counters),
 213                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 214 #endif
 215                 if (err)
 216                         return rte_flow_error_set
 217                                 (error, err,
 218                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 219                                  NULL,
 220                                  "cannot read counter");
 221                 qc->hits_set = 1;
 222                 qc->bytes_set = 1;
 223                 qc->hits = counters[0] - flow->counter->hits;
 224                 qc->bytes = counters[1] - flow->counter->bytes;
 225                 if (qc->reset) {
 226                         flow->counter->hits = counters[0];
 227                         flow->counter->bytes = counters[1];
 228                 }
 229                 return 0;
 230         }
 231         return rte_flow_error_set(error, EINVAL,
 232                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 233                                   NULL,
 234                                   "flow does not have counter");
 235 #else
 236         (void)flow;
 237         (void)data;
 238         return rte_flow_error_set(error, ENOTSUP,
 239                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 240                                   NULL,
 241                                   "counters are not available");
 242 #endif
 243 }
 244
 245 /**
 246  * Add a verbs item specification into @p verbs.
 247  *
 248  * @param[out] verbs
 249  *   Pointer to verbs structure.
 250  * @param[in] src
 251  *   Create specification.
 252  * @param[in] size
 253  *   Size in bytes of the specification to copy.
 254  */
 255 static void
 256 flow_verbs_spec_add(struct mlx5_flow_verbs *verbs, void *src, unsigned int size)
 257 {
 258         void *dst;
 259
 260         if (!verbs)
 261                 return;
 262         MLX5_ASSERT(verbs->specs);
 263         dst = (void *)(verbs->specs + verbs->size);
 264         memcpy(dst, src, size);
 265         ++verbs->attr->num_of_specs;
 266         verbs->size += size;
 267 }
 268
 269 /**
 270  * Convert the @p item into a Verbs specification. This function assumes that
 271  * the input is valid and that there is space to insert the requested item
 272  * into the flow.
 273  *
 274  * @param[in, out] dev_flow
 275  *   Pointer to dev_flow structure.
 276  * @param[in] item
 277  *   Item specification.
 278  * @param[in] item_flags
 279  *   Parsed item flags.
 280  */
 281 static void
 282 flow_verbs_translate_item_eth(struct mlx5_flow *dev_flow,
 283                               const struct rte_flow_item *item,
 284                               uint64_t item_flags)
 285 {
 286         const struct rte_flow_item_eth *spec = item->spec;
 287         const struct rte_flow_item_eth *mask = item->mask;
 288         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 289         struct ibv_flow_spec_eth eth = {
 290                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 291                 .size = size,
 292         };
 293
 294         if (!mask)
 295                 mask = &rte_flow_item_eth_mask;
 296         if (spec) {
 297                 unsigned int i;
 298
 299                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes,
 300                         RTE_ETHER_ADDR_LEN);
 301                 memcpy(&eth.val.src_mac, spec->src.addr_bytes,
 302                         RTE_ETHER_ADDR_LEN);
 303                 eth.val.ether_type = spec->type;
 304                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes,
 305                         RTE_ETHER_ADDR_LEN);
 306                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes,
 307                         RTE_ETHER_ADDR_LEN);
 308                 eth.mask.ether_type = mask->type;
 309                 /* Remove unwanted bits from values. */
 310                 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) {
 311                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 312                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 313                 }
 314                 eth.val.ether_type &= eth.mask.ether_type;
 315         }
 316         flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 317 }
 318
 319 /**
 320  * Update the VLAN tag in the Verbs Ethernet specification.
 321  * This function assumes that the input is valid and there is space to add
 322  * the requested item.
 323  *
 324  * @param[in, out] attr
 325  *   Pointer to Verbs attributes structure.
 326  * @param[in] eth
 327  *   Verbs structure containing the VLAN information to copy.
 328  */
 329 static void
 330 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 331                             struct ibv_flow_spec_eth *eth)
 332 {
 333         unsigned int i;
 334         const enum ibv_flow_spec_type search = eth->type;
 335         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 336                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 337
 338         for (i = 0; i != attr->num_of_specs; ++i) {
 339                 if (hdr->type == search) {
 340                         struct ibv_flow_spec_eth *e =
 341                                 (struct ibv_flow_spec_eth *)hdr;
 342
 343                         e->val.vlan_tag = eth->val.vlan_tag;
 344                         e->mask.vlan_tag = eth->mask.vlan_tag;
 345                         e->val.ether_type = eth->val.ether_type;
 346                         e->mask.ether_type = eth->mask.ether_type;
 347                         break;
 348                 }
 349                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 350         }
 351 }
 352
 353 /**
 354  * Convert the @p item into a Verbs specification. This function assumes that
 355  * the input is valid and that there is space to insert the requested item
 356  * into the flow.
 357  *
 358  * @param[in, out] dev_flow
 359  *   Pointer to dev_flow structure.
 360  * @param[in] item
 361  *   Item specification.
 362  * @param[in] item_flags
 363  *   Parsed item flags.
 364  */
 365 static void
 366 flow_verbs_translate_item_vlan(struct mlx5_flow *dev_flow,
 367                                const struct rte_flow_item *item,
 368                                uint64_t item_flags)
 369 {
 370         const struct rte_flow_item_vlan *spec = item->spec;
 371         const struct rte_flow_item_vlan *mask = item->mask;
 372         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 373         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
 374         struct ibv_flow_spec_eth eth = {
 375                 .type = IBV_FLOW_SPEC_ETH | VERBS_SPEC_INNER(item_flags),
 376                 .size = size,
 377         };
 378         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 379                                       MLX5_FLOW_LAYER_OUTER_L2;
 380
 381         if (!mask)
 382                 mask = &rte_flow_item_vlan_mask;
 383         if (spec) {
 384                 eth.val.vlan_tag = spec->tci;
 385                 eth.mask.vlan_tag = mask->tci;
 386                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 387                 eth.val.ether_type = spec->inner_type;
 388                 eth.mask.ether_type = mask->inner_type;
 389                 eth.val.ether_type &= eth.mask.ether_type;
 390         }
 391         if (!(item_flags & l2m))
 392                 flow_verbs_spec_add(&dev_flow->verbs, &eth, size);
 393         else
 394                 flow_verbs_item_vlan_update(dev_flow->verbs.attr, &eth);
 395         if (!tunnel)
 396                 dev_flow->verbs.vf_vlan.tag =
 397                         rte_be_to_cpu_16(spec->tci) & 0x0fff;
 398 }
 399
 400 /**
 401  * Convert the @p item into a Verbs specification. This function assumes that
 402  * the input is valid and that there is space to insert the requested item
 403  * into the flow.
 404  *
 405  * @param[in, out] dev_flow
 406  *   Pointer to dev_flow structure.
 407  * @param[in] item
 408  *   Item specification.
 409  * @param[in] item_flags
 410  *   Parsed item flags.
 411  */
 412 static void
 413 flow_verbs_translate_item_ipv4(struct mlx5_flow *dev_flow,
 414                                const struct rte_flow_item *item,
 415                                uint64_t item_flags)
 416 {
 417         const struct rte_flow_item_ipv4 *spec = item->spec;
 418         const struct rte_flow_item_ipv4 *mask = item->mask;
 419         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 420         struct ibv_flow_spec_ipv4_ext ipv4 = {
 421                 .type = IBV_FLOW_SPEC_IPV4_EXT | VERBS_SPEC_INNER(item_flags),
 422                 .size = size,
 423         };
 424
 425         if (!mask)
 426                 mask = &rte_flow_item_ipv4_mask;
 427         if (spec) {
 428                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 429                         .src_ip = spec->hdr.src_addr,
 430                         .dst_ip = spec->hdr.dst_addr,
 431                         .proto = spec->hdr.next_proto_id,
 432                         .tos = spec->hdr.type_of_service,
 433                 };
 434                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 435                         .src_ip = mask->hdr.src_addr,
 436                         .dst_ip = mask->hdr.dst_addr,
 437                         .proto = mask->hdr.next_proto_id,
 438                         .tos = mask->hdr.type_of_service,
 439                 };
 440                 /* Remove unwanted bits from values. */
 441                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 442                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 443                 ipv4.val.proto &= ipv4.mask.proto;
 444                 ipv4.val.tos &= ipv4.mask.tos;
 445         }
 446         flow_verbs_spec_add(&dev_flow->verbs, &ipv4, size);
 447 }
 448
 449 /**
 450  * Convert the @p item into a Verbs specification. This function assumes that
 451  * the input is valid and that there is space to insert the requested item
 452  * into the flow.
 453  *
 454  * @param[in, out] dev_flow
 455  *   Pointer to dev_flow structure.
 456  * @param[in] item
 457  *   Item specification.
 458  * @param[in] item_flags
 459  *   Parsed item flags.
 460  */
 461 static void
 462 flow_verbs_translate_item_ipv6(struct mlx5_flow *dev_flow,
 463                                const struct rte_flow_item *item,
 464                                uint64_t item_flags)
 465 {
 466         const struct rte_flow_item_ipv6 *spec = item->spec;
 467         const struct rte_flow_item_ipv6 *mask = item->mask;
 468         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 469         struct ibv_flow_spec_ipv6 ipv6 = {
 470                 .type = IBV_FLOW_SPEC_IPV6 | VERBS_SPEC_INNER(item_flags),
 471                 .size = size,
 472         };
 473
 474         if (!mask)
 475                 mask = &rte_flow_item_ipv6_mask;
 476         if (spec) {
 477                 unsigned int i;
 478                 uint32_t vtc_flow_val;
 479                 uint32_t vtc_flow_mask;
 480
 481                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 482                        RTE_DIM(ipv6.val.src_ip));
 483                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 484                        RTE_DIM(ipv6.val.dst_ip));
 485                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 486                        RTE_DIM(ipv6.mask.src_ip));
 487                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 488                        RTE_DIM(ipv6.mask.dst_ip));
 489                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 490                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 491                 ipv6.val.flow_label =
 492                         rte_cpu_to_be_32((vtc_flow_val & RTE_IPV6_HDR_FL_MASK) >>
 493                                          RTE_IPV6_HDR_FL_SHIFT);
 494                 ipv6.val.traffic_class = (vtc_flow_val & RTE_IPV6_HDR_TC_MASK) >>
 495                                          RTE_IPV6_HDR_TC_SHIFT;
 496                 ipv6.val.next_hdr = spec->hdr.proto;
 497                 ipv6.mask.flow_label =
 498                         rte_cpu_to_be_32((vtc_flow_mask & RTE_IPV6_HDR_FL_MASK) >>
 499                                          RTE_IPV6_HDR_FL_SHIFT);
 500                 ipv6.mask.traffic_class = (vtc_flow_mask & RTE_IPV6_HDR_TC_MASK) >>
 501                                           RTE_IPV6_HDR_TC_SHIFT;
 502                 ipv6.mask.next_hdr = mask->hdr.proto;
 503                 /* Remove unwanted bits from values. */
 504                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 505                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 506                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 507                 }
 508                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 509                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 510                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 511         }
 512         flow_verbs_spec_add(&dev_flow->verbs, &ipv6, size);
 513 }
 514
 515 /**
 516  * Convert the @p item into a Verbs specification. This function assumes that
 517  * the input is valid and that there is space to insert the requested item
 518  * into the flow.
 519  *
 520  * @param[in, out] dev_flow
 521  *   Pointer to dev_flow structure.
 522  * @param[in] item
 523  *   Item specification.
 524  * @param[in] item_flags
 525  *   Parsed item flags.
 526  */
 527 static void
 528 flow_verbs_translate_item_tcp(struct mlx5_flow *dev_flow,
 529                               const struct rte_flow_item *item,
 530                               uint64_t item_flags __rte_unused)
 531 {
 532         const struct rte_flow_item_tcp *spec = item->spec;
 533         const struct rte_flow_item_tcp *mask = item->mask;
 534         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 535         struct ibv_flow_spec_tcp_udp tcp = {
 536                 .type = IBV_FLOW_SPEC_TCP | VERBS_SPEC_INNER(item_flags),
 537                 .size = size,
 538         };
 539
 540         if (!mask)
 541                 mask = &rte_flow_item_tcp_mask;
 542         if (spec) {
 543                 tcp.val.dst_port = spec->hdr.dst_port;
 544                 tcp.val.src_port = spec->hdr.src_port;
 545                 tcp.mask.dst_port = mask->hdr.dst_port;
 546                 tcp.mask.src_port = mask->hdr.src_port;
 547                 /* Remove unwanted bits from values. */
 548                 tcp.val.src_port &= tcp.mask.src_port;
 549                 tcp.val.dst_port &= tcp.mask.dst_port;
 550         }
 551         flow_verbs_spec_add(&dev_flow->verbs, &tcp, size);
 552 }
 553
 554 /**
 555  * Convert the @p item into a Verbs specification. This function assumes that
 556  * the input is valid and that there is space to insert the requested item
 557  * into the flow.
 558  *
 559  * @param[in, out] dev_flow
 560  *   Pointer to dev_flow structure.
 561  * @param[in] item
 562  *   Item specification.
 563  * @param[in] item_flags
 564  *   Parsed item flags.
 565  */
 566 static void
 567 flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow,
 568                               const struct rte_flow_item *item,
 569                               uint64_t item_flags __rte_unused)
 570 {
 571         const struct rte_flow_item_udp *spec = item->spec;
 572         const struct rte_flow_item_udp *mask = item->mask;
 573         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 574         struct ibv_flow_spec_tcp_udp udp = {
 575                 .type = IBV_FLOW_SPEC_UDP | VERBS_SPEC_INNER(item_flags),
 576                 .size = size,
 577         };
 578
 579         if (!mask)
 580                 mask = &rte_flow_item_udp_mask;
 581         if (spec) {
 582                 udp.val.dst_port = spec->hdr.dst_port;
 583                 udp.val.src_port = spec->hdr.src_port;
 584                 udp.mask.dst_port = mask->hdr.dst_port;
 585                 udp.mask.src_port = mask->hdr.src_port;
 586                 /* Remove unwanted bits from values. */
 587                 udp.val.src_port &= udp.mask.src_port;
 588                 udp.val.dst_port &= udp.mask.dst_port;
 589         }
 590         flow_verbs_spec_add(&dev_flow->verbs, &udp, size);
 591 }
 592
 593 /**
 594  * Convert the @p item into a Verbs specification. This function assumes that
 595  * the input is valid and that there is space to insert the requested item
 596  * into the flow.
 597  *
 598  * @param[in, out] dev_flow
 599  *   Pointer to dev_flow structure.
 600  * @param[in] item
 601  *   Item specification.
 602  * @param[in] item_flags
 603  *   Parsed item flags.
 604  */
 605 static void
 606 flow_verbs_translate_item_vxlan(struct mlx5_flow *dev_flow,
 607                                 const struct rte_flow_item *item,
 608                                 uint64_t item_flags __rte_unused)
 609 {
 610         const struct rte_flow_item_vxlan *spec = item->spec;
 611         const struct rte_flow_item_vxlan *mask = item->mask;
 612         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 613         struct ibv_flow_spec_tunnel vxlan = {
 614                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 615                 .size = size,
 616         };
 617         union vni {
 618                 uint32_t vlan_id;
 619                 uint8_t vni[4];
 620         } id = { .vlan_id = 0, };
 621
 622         if (!mask)
 623                 mask = &rte_flow_item_vxlan_mask;
 624         if (spec) {
 625                 memcpy(&id.vni[1], spec->vni, 3);
 626                 vxlan.val.tunnel_id = id.vlan_id;
 627                 memcpy(&id.vni[1], mask->vni, 3);
 628                 vxlan.mask.tunnel_id = id.vlan_id;
 629                 /* Remove unwanted bits from values. */
 630                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 631         }
 632         flow_verbs_spec_add(&dev_flow->verbs, &vxlan, size);
 633 }
 634
 635 /**
 636  * Convert the @p item into a Verbs specification. This function assumes that
 637  * the input is valid and that there is space to insert the requested item
 638  * into the flow.
 639  *
 640  * @param[in, out] dev_flow
 641  *   Pointer to dev_flow structure.
 642  * @param[in] item
 643  *   Item specification.
 644  * @param[in] item_flags
 645  *   Parsed item flags.
 646  */
 647 static void
 648 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow *dev_flow,
 649                                     const struct rte_flow_item *item,
 650                                     uint64_t item_flags __rte_unused)
 651 {
 652         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 653         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 654         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 655         struct ibv_flow_spec_tunnel vxlan_gpe = {
 656                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 657                 .size = size,
 658         };
 659         union vni {
 660                 uint32_t vlan_id;
 661                 uint8_t vni[4];
 662         } id = { .vlan_id = 0, };
 663
 664         if (!mask)
 665                 mask = &rte_flow_item_vxlan_gpe_mask;
 666         if (spec) {
 667                 memcpy(&id.vni[1], spec->vni, 3);
 668                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 669                 memcpy(&id.vni[1], mask->vni, 3);
 670                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 671                 /* Remove unwanted bits from values. */
 672                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 673         }
 674         flow_verbs_spec_add(&dev_flow->verbs, &vxlan_gpe, size);
 675 }
 676
 677 /**
 678  * Update the protocol in Verbs IPv4/IPv6 spec.
 679  *
 680  * @param[in, out] attr
 681  *   Pointer to Verbs attributes structure.
 682  * @param[in] search
 683  *   Specification type to search in order to update the IP protocol.
 684  * @param[in] protocol
 685  *   Protocol value to set if none is present in the specification.
 686  */
 687 static void
 688 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 689                                        enum ibv_flow_spec_type search,
 690                                        uint8_t protocol)
 691 {
 692         unsigned int i;
 693         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 694                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 695
 696         if (!attr)
 697                 return;
 698         for (i = 0; i != attr->num_of_specs; ++i) {
 699                 if (hdr->type == search) {
 700                         union {
 701                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 702                                 struct ibv_flow_spec_ipv6 *ipv6;
 703                         } ip;
 704
 705                         switch (search) {
 706                         case IBV_FLOW_SPEC_IPV4_EXT:
 707                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 708                                 if (!ip.ipv4->val.proto) {
 709                                         ip.ipv4->val.proto = protocol;
 710                                         ip.ipv4->mask.proto = 0xff;
 711                                 }
 712                                 break;
 713                         case IBV_FLOW_SPEC_IPV6:
 714                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 715                                 if (!ip.ipv6->val.next_hdr) {
 716                                         ip.ipv6->val.next_hdr = protocol;
 717                                         ip.ipv6->mask.next_hdr = 0xff;
 718                                 }
 719                                 break;
 720                         default:
 721                                 break;
 722                         }
 723                         break;
 724                 }
 725                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 726         }
 727 }
 728
 729 /**
 730  * Convert the @p item into a Verbs specification. This function assumes that
 731  * the input is valid and that there is space to insert the requested item
 732  * into the flow.
 733  *
 734  * @param[in, out] dev_flow
 735  *   Pointer to dev_flow structure.
 736  * @param[in] item
 737  *   Item specification.
 738  * @param[in] item_flags
 739  *   Parsed item flags.
 740  */
 741 static void
 742 flow_verbs_translate_item_gre(struct mlx5_flow *dev_flow,
 743                               const struct rte_flow_item *item __rte_unused,
 744                               uint64_t item_flags)
 745 {
 746         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
 747 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 748         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 749         struct ibv_flow_spec_tunnel tunnel = {
 750                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 751                 .size = size,
 752         };
 753 #else
 754         const struct rte_flow_item_gre *spec = item->spec;
 755         const struct rte_flow_item_gre *mask = item->mask;
 756         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 757         struct ibv_flow_spec_gre tunnel = {
 758                 .type = IBV_FLOW_SPEC_GRE,
 759                 .size = size,
 760         };
 761
 762         if (!mask)
 763                 mask = &rte_flow_item_gre_mask;
 764         if (spec) {
 765                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 766                 tunnel.val.protocol = spec->protocol;
 767                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 768                 tunnel.mask.protocol = mask->protocol;
 769                 /* Remove unwanted bits from values. */
 770                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 771                 tunnel.val.protocol &= tunnel.mask.protocol;
 772                 tunnel.val.key &= tunnel.mask.key;
 773         }
 774 #endif
 775         if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 776                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 777                                                        IBV_FLOW_SPEC_IPV4_EXT,
 778                                                        IPPROTO_GRE);
 779         else
 780                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 781                                                        IBV_FLOW_SPEC_IPV6,
 782                                                        IPPROTO_GRE);
 783         flow_verbs_spec_add(verbs, &tunnel, size);
 784 }
 785
 786 /**
 787  * Convert the @p action into a Verbs specification. This function assumes that
 788  * the input is valid and that there is space to insert the requested action
 789  * into the flow. This function also return the action that was added.
 790  *
 791  * @param[in, out] dev_flow
 792  *   Pointer to dev_flow structure.
 793  * @param[in] item
 794  *   Item specification.
 795  * @param[in] item_flags
 796  *   Parsed item flags.
 797  */
 798 static void
 799 flow_verbs_translate_item_mpls(struct mlx5_flow *dev_flow __rte_unused,
 800                                const struct rte_flow_item *item __rte_unused,
 801                                uint64_t item_flags __rte_unused)
 802 {
 803 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 804         const struct rte_flow_item_mpls *spec = item->spec;
 805         const struct rte_flow_item_mpls *mask = item->mask;
 806         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 807         struct ibv_flow_spec_mpls mpls = {
 808                 .type = IBV_FLOW_SPEC_MPLS,
 809                 .size = size,
 810         };
 811
 812         if (!mask)
 813                 mask = &rte_flow_item_mpls_mask;
 814         if (spec) {
 815                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 816                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 817                 /* Remove unwanted bits from values.  */
 818                 mpls.val.label &= mpls.mask.label;
 819         }
 820         flow_verbs_spec_add(&dev_flow->verbs, &mpls, size);
 821 #endif
 822 }
 823
 824 /**
 825  * Convert the @p action into a Verbs specification. This function assumes that
 826  * the input is valid and that there is space to insert the requested action
 827  * into the flow.
 828  *
 829  * @param[in] dev_flow
 830  *   Pointer to mlx5_flow.
 831  * @param[in] action
 832  *   Action configuration.
 833  */
 834 static void
 835 flow_verbs_translate_action_drop
 836         (struct mlx5_flow *dev_flow,
 837          const struct rte_flow_action *action __rte_unused)
 838 {
 839         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 840         struct ibv_flow_spec_action_drop drop = {
 841                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 842                         .size = size,
 843         };
 844
 845         flow_verbs_spec_add(&dev_flow->verbs, &drop, size);
 846 }
 847
 848 /**
 849  * Convert the @p action into a Verbs specification. This function assumes that
 850  * the input is valid and that there is space to insert the requested action
 851  * into the flow.
 852  *
 853  * @param[in] dev_flow
 854  *   Pointer to mlx5_flow.
 855  * @param[in] action
 856  *   Action configuration.
 857  */
 858 static void
 859 flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow,
 860                                   const struct rte_flow_action *action)
 861 {
 862         const struct rte_flow_action_queue *queue = action->conf;
 863         struct rte_flow *flow = dev_flow->flow;
 864
 865         if (flow->rss.queue)
 866                 (*flow->rss.queue)[0] = queue->index;
 867         flow->rss.queue_num = 1;
 868 }
 869
 870 /**
 871  * Convert the @p action into a Verbs specification. This function assumes that
 872  * the input is valid and that there is space to insert the requested action
 873  * into the flow.
 874  *
 875  * @param[in] action
 876  *   Action configuration.
 877  * @param[in, out] action_flags
 878  *   Pointer to the detected actions.
 879  * @param[in] dev_flow
 880  *   Pointer to mlx5_flow.
 881  */
 882 static void
 883 flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow,
 884                                 const struct rte_flow_action *action)
 885 {
 886         const struct rte_flow_action_rss *rss = action->conf;
 887         const uint8_t *rss_key;
 888         struct rte_flow *flow = dev_flow->flow;
 889
 890         if (flow->rss.queue)
 891                 memcpy((*flow->rss.queue), rss->queue,
 892                        rss->queue_num * sizeof(uint16_t));
 893         flow->rss.queue_num = rss->queue_num;
 894         /* NULL RSS key indicates default RSS key. */
 895         rss_key = !rss->key ? rss_hash_default_key : rss->key;
 896         memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN);
 897         /*
 898          * rss->level and rss.types should be set in advance when expanding
 899          * items for RSS.
 900          */
 901 }
 902
 903 /**
 904  * Convert the @p action into a Verbs specification. This function assumes that
 905  * the input is valid and that there is space to insert the requested action
 906  * into the flow.
 907  *
 908  * @param[in] dev_flow
 909  *   Pointer to mlx5_flow.
 910  * @param[in] action
 911  *   Action configuration.
 912  */
 913 static void
 914 flow_verbs_translate_action_flag
 915         (struct mlx5_flow *dev_flow,
 916          const struct rte_flow_action *action __rte_unused)
 917 {
 918         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 919         struct ibv_flow_spec_action_tag tag = {
 920                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 921                 .size = size,
 922                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 923         };
 924
 925         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 926 }
 927
 928 /**
 929  * Convert the @p action into a Verbs specification. This function assumes that
 930  * the input is valid and that there is space to insert the requested action
 931  * into the flow.
 932  *
 933  * @param[in] dev_flow
 934  *   Pointer to mlx5_flow.
 935  * @param[in] action
 936  *   Action configuration.
 937  */
 938 static void
 939 flow_verbs_translate_action_mark(struct mlx5_flow *dev_flow,
 940                                  const struct rte_flow_action *action)
 941 {
 942         const struct rte_flow_action_mark *mark = action->conf;
 943         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 944         struct ibv_flow_spec_action_tag tag = {
 945                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 946                 .size = size,
 947                 .tag_id = mlx5_flow_mark_set(mark->id),
 948         };
 949
 950         flow_verbs_spec_add(&dev_flow->verbs, &tag, size);
 951 }
 952
 953 /**
 954  * Convert the @p action into a Verbs specification. This function assumes that
 955  * the input is valid and that there is space to insert the requested action
 956  * into the flow.
 957  *
 958  * @param[in] dev
 959  *   Pointer to the Ethernet device structure.
 960  * @param[in] action
 961  *   Action configuration.
 962  * @param[in] dev_flow
 963  *   Pointer to mlx5_flow.
 964  * @param[out] error
 965  *   Pointer to error structure.
 966  *
 967  * @return
 968  *   0 On success else a negative errno value is returned and rte_errno is set.
 969  */
 970 static int
 971 flow_verbs_translate_action_count(struct mlx5_flow *dev_flow,
 972                                   const struct rte_flow_action *action,
 973                                   struct rte_eth_dev *dev,
 974                                   struct rte_flow_error *error)
 975 {
 976         const struct rte_flow_action_count *count = action->conf;
 977         struct rte_flow *flow = dev_flow->flow;
 978 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 979         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 980         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 981         struct ibv_flow_spec_counter_action counter = {
 982                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
 983                 .size = size,
 984         };
 985 #endif
 986
 987         if (!flow->counter) {
 988                 flow->counter = flow_verbs_counter_new(dev, count->shared,
 989                                                        count->id);
 990                 if (!flow->counter)
 991                         return rte_flow_error_set(error, rte_errno,
 992                                                   RTE_FLOW_ERROR_TYPE_ACTION,
 993                                                   action,
 994                                                   "cannot get counter"
 995                                                   " context.");
 996         }
 997 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 998         counter.counter_set_handle = flow->counter->cs->handle;
 999         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1000 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1001         counter.counters = flow->counter->cs;
1002         flow_verbs_spec_add(&dev_flow->verbs, &counter, size);
1003 #endif
1004         return 0;
1005 }
1006
1007 /**
1008  * Internal validation function. For validating both actions and items.
1009  *
1010  * @param[in] dev
1011  *   Pointer to the Ethernet device structure.
1012  * @param[in] attr
1013  *   Pointer to the flow attributes.
1014  * @param[in] items
1015  *   Pointer to the list of items.
1016  * @param[in] actions
1017  *   Pointer to the list of actions.
1018  * @param[in] external
1019  *   This flow rule is created by request external to PMD.
1020  * @param[out] error
1021  *   Pointer to the error structure.
1022  *
1023  * @return
1024  *   0 on success, a negative errno value otherwise and rte_errno is set.
1025  */
1026 static int
1027 flow_verbs_validate(struct rte_eth_dev *dev,
1028                     const struct rte_flow_attr *attr,
1029                     const struct rte_flow_item items[],
1030                     const struct rte_flow_action actions[],
1031                     bool external __rte_unused,
1032                     struct rte_flow_error *error)
1033 {
1034         int ret;
1035         uint64_t action_flags = 0;
1036         uint64_t item_flags = 0;
1037         uint64_t last_item = 0;
1038         uint8_t next_protocol = 0xff;
1039         uint16_t ether_type = 0;
1040
1041         if (items == NULL)
1042                 return -1;
1043         ret = mlx5_flow_validate_attributes(dev, attr, error);
1044         if (ret < 0)
1045                 return ret;
1046         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1047                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1048                 int ret = 0;
1049
1050                 switch (items->type) {
1051                 case RTE_FLOW_ITEM_TYPE_VOID:
1052                         break;
1053                 case RTE_FLOW_ITEM_TYPE_ETH:
1054                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1055                                                           error);
1056                         if (ret < 0)
1057                                 return ret;
1058                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1059                                              MLX5_FLOW_LAYER_OUTER_L2;
1060                         if (items->mask != NULL && items->spec != NULL) {
1061                                 ether_type =
1062                                         ((const struct rte_flow_item_eth *)
1063                                          items->spec)->type;
1064                                 ether_type &=
1065                                         ((const struct rte_flow_item_eth *)
1066                                          items->mask)->type;
1067                                 ether_type = rte_be_to_cpu_16(ether_type);
1068                         } else {
1069                                 ether_type = 0;
1070                         }
1071                         break;
1072                 case RTE_FLOW_ITEM_TYPE_VLAN:
1073                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1074                                                            dev, error);
1075                         if (ret < 0)
1076                                 return ret;
1077                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1078                                               MLX5_FLOW_LAYER_INNER_VLAN) :
1079                                              (MLX5_FLOW_LAYER_OUTER_L2 |
1080                                               MLX5_FLOW_LAYER_OUTER_VLAN);
1081                         if (items->mask != NULL && items->spec != NULL) {
1082                                 ether_type =
1083                                         ((const struct rte_flow_item_vlan *)
1084                                          items->spec)->inner_type;
1085                                 ether_type &=
1086                                         ((const struct rte_flow_item_vlan *)
1087                                          items->mask)->inner_type;
1088                                 ether_type = rte_be_to_cpu_16(ether_type);
1089                         } else {
1090                                 ether_type = 0;
1091                         }
1092                         break;
1093                 case RTE_FLOW_ITEM_TYPE_IPV4:
1094                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1095                                                            last_item,
1096                                                            ether_type, NULL,
1097                                                            error);
1098                         if (ret < 0)
1099                                 return ret;
1100                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1101                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1102                         if (items->mask != NULL &&
1103                             ((const struct rte_flow_item_ipv4 *)
1104                              items->mask)->hdr.next_proto_id) {
1105                                 next_protocol =
1106                                         ((const struct rte_flow_item_ipv4 *)
1107                                          (items->spec))->hdr.next_proto_id;
1108                                 next_protocol &=
1109                                         ((const struct rte_flow_item_ipv4 *)
1110                                          (items->mask))->hdr.next_proto_id;
1111                         } else {
1112                                 /* Reset for inner layer. */
1113                                 next_protocol = 0xff;
1114                         }
1115                         break;
1116                 case RTE_FLOW_ITEM_TYPE_IPV6:
1117                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1118                                                            last_item,
1119                                                            ether_type, NULL,
1120                                                            error);
1121                         if (ret < 0)
1122                                 return ret;
1123                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1124                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1125                         if (items->mask != NULL &&
1126                             ((const struct rte_flow_item_ipv6 *)
1127                              items->mask)->hdr.proto) {
1128                                 next_protocol =
1129                                         ((const struct rte_flow_item_ipv6 *)
1130                                          items->spec)->hdr.proto;
1131                                 next_protocol &=
1132                                         ((const struct rte_flow_item_ipv6 *)
1133                                          items->mask)->hdr.proto;
1134                         } else {
1135                                 /* Reset for inner layer. */
1136                                 next_protocol = 0xff;
1137                         }
1138                         break;
1139                 case RTE_FLOW_ITEM_TYPE_UDP:
1140                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1141                                                           next_protocol,
1142                                                           error);
1143                         if (ret < 0)
1144                                 return ret;
1145                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1146                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
1147                         break;
1148                 case RTE_FLOW_ITEM_TYPE_TCP:
1149                         ret = mlx5_flow_validate_item_tcp
1150                                                 (items, item_flags,
1151                                                  next_protocol,
1152                                                  &rte_flow_item_tcp_mask,
1153                                                  error);
1154                         if (ret < 0)
1155                                 return ret;
1156                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1157                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
1158                         break;
1159                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1160                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1161                                                             error);
1162                         if (ret < 0)
1163                                 return ret;
1164                         last_item = MLX5_FLOW_LAYER_VXLAN;
1165                         break;
1166                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1167                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1168                                                                 item_flags,
1169                                                                 dev, error);
1170                         if (ret < 0)
1171                                 return ret;
1172                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
1173                         break;
1174                 case RTE_FLOW_ITEM_TYPE_GRE:
1175                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1176                                                           next_protocol, error);
1177                         if (ret < 0)
1178                                 return ret;
1179                         last_item = MLX5_FLOW_LAYER_GRE;
1180                         break;
1181                 case RTE_FLOW_ITEM_TYPE_MPLS:
1182                         ret = mlx5_flow_validate_item_mpls(dev, items,
1183                                                            item_flags,
1184                                                            last_item, error);
1185                         if (ret < 0)
1186                                 return ret;
1187                         last_item = MLX5_FLOW_LAYER_MPLS;
1188                         break;
1189                 default:
1190                         return rte_flow_error_set(error, ENOTSUP,
1191                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1192                                                   NULL, "item not supported");
1193                 }
1194                 item_flags |= last_item;
1195         }
1196         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1197                 switch (actions->type) {
1198                 case RTE_FLOW_ACTION_TYPE_VOID:
1199                         break;
1200                 case RTE_FLOW_ACTION_TYPE_FLAG:
1201                         ret = mlx5_flow_validate_action_flag(action_flags,
1202                                                              attr,
1203                                                              error);
1204                         if (ret < 0)
1205                                 return ret;
1206                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1207                         break;
1208                 case RTE_FLOW_ACTION_TYPE_MARK:
1209                         ret = mlx5_flow_validate_action_mark(actions,
1210                                                              action_flags,
1211                                                              attr,
1212                                                              error);
1213                         if (ret < 0)
1214                                 return ret;
1215                         action_flags |= MLX5_FLOW_ACTION_MARK;
1216                         break;
1217                 case RTE_FLOW_ACTION_TYPE_DROP:
1218                         ret = mlx5_flow_validate_action_drop(action_flags,
1219                                                              attr,
1220                                                              error);
1221                         if (ret < 0)
1222                                 return ret;
1223                         action_flags |= MLX5_FLOW_ACTION_DROP;
1224                         break;
1225                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1226                         ret = mlx5_flow_validate_action_queue(actions,
1227                                                               action_flags, dev,
1228                                                               attr,
1229                                                               error);
1230                         if (ret < 0)
1231                                 return ret;
1232                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1233                         break;
1234                 case RTE_FLOW_ACTION_TYPE_RSS:
1235                         ret = mlx5_flow_validate_action_rss(actions,
1236                                                             action_flags, dev,
1237                                                             attr, item_flags,
1238                                                             error);
1239                         if (ret < 0)
1240                                 return ret;
1241                         action_flags |= MLX5_FLOW_ACTION_RSS;
1242                         break;
1243                 case RTE_FLOW_ACTION_TYPE_COUNT:
1244                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1245                         if (ret < 0)
1246                                 return ret;
1247                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1248                         break;
1249                 default:
1250                         return rte_flow_error_set(error, ENOTSUP,
1251                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1252                                                   actions,
1253                                                   "action not supported");
1254                 }
1255         }
1256         /*
1257          * Validate the drop action mutual exclusion with other actions.
1258          * Drop action is mutually-exclusive with any other action, except for
1259          * Count action.
1260          */
1261         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
1262             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
1263                 return rte_flow_error_set(error, EINVAL,
1264                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1265                                           "Drop action is mutually-exclusive "
1266                                           "with any other action, except for "
1267                                           "Count action");
1268         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1269                 return rte_flow_error_set(error, EINVAL,
1270                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1271                                           "no fate action is found");
1272         return 0;
1273 }
1274
1275 /**
1276  * Calculate the required bytes that are needed for the action part of the verbs
1277  * flow.
1278  *
1279  * @param[in] actions
1280  *   Pointer to the list of actions.
1281  *
1282  * @return
1283  *   The size of the memory needed for all actions.
1284  */
1285 static int
1286 flow_verbs_get_actions_size(const struct rte_flow_action actions[])
1287 {
1288         int size = 0;
1289
1290         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1291                 switch (actions->type) {
1292                 case RTE_FLOW_ACTION_TYPE_VOID:
1293                         break;
1294                 case RTE_FLOW_ACTION_TYPE_FLAG:
1295                         size += sizeof(struct ibv_flow_spec_action_tag);
1296                         break;
1297                 case RTE_FLOW_ACTION_TYPE_MARK:
1298                         size += sizeof(struct ibv_flow_spec_action_tag);
1299                         break;
1300                 case RTE_FLOW_ACTION_TYPE_DROP:
1301                         size += sizeof(struct ibv_flow_spec_action_drop);
1302                         break;
1303                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1304                         break;
1305                 case RTE_FLOW_ACTION_TYPE_RSS:
1306                         break;
1307                 case RTE_FLOW_ACTION_TYPE_COUNT:
1308 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1309         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1310                         size += sizeof(struct ibv_flow_spec_counter_action);
1311 #endif
1312                         break;
1313                 default:
1314                         break;
1315                 }
1316         }
1317         return size;
1318 }
1319
1320 /**
1321  * Calculate the required bytes that are needed for the item part of the verbs
1322  * flow.
1323  *
1324  * @param[in] items
1325  *   Pointer to the list of items.
1326  *
1327  * @return
1328  *   The size of the memory needed for all items.
1329  */
1330 static int
1331 flow_verbs_get_items_size(const struct rte_flow_item items[])
1332 {
1333         int size = 0;
1334
1335         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1336                 switch (items->type) {
1337                 case RTE_FLOW_ITEM_TYPE_VOID:
1338                         break;
1339                 case RTE_FLOW_ITEM_TYPE_ETH:
1340                         size += sizeof(struct ibv_flow_spec_eth);
1341                         break;
1342                 case RTE_FLOW_ITEM_TYPE_VLAN:
1343                         size += sizeof(struct ibv_flow_spec_eth);
1344                         break;
1345                 case RTE_FLOW_ITEM_TYPE_IPV4:
1346                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1347                         break;
1348                 case RTE_FLOW_ITEM_TYPE_IPV6:
1349                         size += sizeof(struct ibv_flow_spec_ipv6);
1350                         break;
1351                 case RTE_FLOW_ITEM_TYPE_UDP:
1352                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1353                         break;
1354                 case RTE_FLOW_ITEM_TYPE_TCP:
1355                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1356                         break;
1357                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1358                         size += sizeof(struct ibv_flow_spec_tunnel);
1359                         break;
1360                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1361                         size += sizeof(struct ibv_flow_spec_tunnel);
1362                         break;
1363 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1364                 case RTE_FLOW_ITEM_TYPE_GRE:
1365                         size += sizeof(struct ibv_flow_spec_gre);
1366                         break;
1367                 case RTE_FLOW_ITEM_TYPE_MPLS:
1368                         size += sizeof(struct ibv_flow_spec_mpls);
1369                         break;
1370 #else
1371                 case RTE_FLOW_ITEM_TYPE_GRE:
1372                         size += sizeof(struct ibv_flow_spec_tunnel);
1373                         break;
1374 #endif
1375                 default:
1376                         break;
1377                 }
1378         }
1379         return size;
1380 }
1381
1382 /**
1383  * Internal preparation function. Allocate mlx5_flow with the required size.
1384  * The required size is calculate based on the actions and items. This function
1385  * also returns the detected actions and items for later use.
1386  *
1387  * @param[in] attr
1388  *   Pointer to the flow attributes.
1389  * @param[in] items
1390  *   Pointer to the list of items.
1391  * @param[in] actions
1392  *   Pointer to the list of actions.
1393  * @param[out] error
1394  *   Pointer to the error structure.
1395  *
1396  * @return
1397  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1398  *   is set.
1399  */
1400 static struct mlx5_flow *
1401 flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
1402                    const struct rte_flow_item items[],
1403                    const struct rte_flow_action actions[],
1404                    struct rte_flow_error *error)
1405 {
1406         size_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
1407         struct mlx5_flow *dev_flow;
1408
1409         size += flow_verbs_get_actions_size(actions);
1410         size += flow_verbs_get_items_size(items);
1411         dev_flow = rte_calloc(__func__, 1, size, 0);
1412         if (!dev_flow) {
1413                 rte_flow_error_set(error, ENOMEM,
1414                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1415                                    "not enough memory to create flow");
1416                 return NULL;
1417         }
1418         dev_flow->verbs.attr = (void *)(dev_flow + 1);
1419         dev_flow->verbs.specs = (void *)(dev_flow->verbs.attr + 1);
1420         dev_flow->ingress = attr->ingress;
1421         dev_flow->transfer = attr->transfer;
1422         return dev_flow;
1423 }
1424
1425 /**
1426  * Fill the flow with verb spec.
1427  *
1428  * @param[in] dev
1429  *   Pointer to Ethernet device.
1430  * @param[in, out] dev_flow
1431  *   Pointer to the mlx5 flow.
1432  * @param[in] attr
1433  *   Pointer to the flow attributes.
1434  * @param[in] items
1435  *   Pointer to the list of items.
1436  * @param[in] actions
1437  *   Pointer to the list of actions.
1438  * @param[out] error
1439  *   Pointer to the error structure.
1440  *
1441  * @return
1442  *   0 on success, else a negative errno value otherwise and rte_errno is set.
1443  */
1444 static int
1445 flow_verbs_translate(struct rte_eth_dev *dev,
1446                      struct mlx5_flow *dev_flow,
1447                      const struct rte_flow_attr *attr,
1448                      const struct rte_flow_item items[],
1449                      const struct rte_flow_action actions[],
1450                      struct rte_flow_error *error)
1451 {
1452         uint64_t item_flags = 0;
1453         uint64_t action_flags = 0;
1454         uint64_t priority = attr->priority;
1455         uint32_t subpriority = 0;
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457
1458         if (priority == MLX5_FLOW_PRIO_RSVD)
1459                 priority = priv->config.flow_prio - 1;
1460         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1461                 int ret;
1462
1463                 switch (actions->type) {
1464                 case RTE_FLOW_ACTION_TYPE_VOID:
1465                         break;
1466                 case RTE_FLOW_ACTION_TYPE_FLAG:
1467                         flow_verbs_translate_action_flag(dev_flow, actions);
1468                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1469                         break;
1470                 case RTE_FLOW_ACTION_TYPE_MARK:
1471                         flow_verbs_translate_action_mark(dev_flow, actions);
1472                         action_flags |= MLX5_FLOW_ACTION_MARK;
1473                         break;
1474                 case RTE_FLOW_ACTION_TYPE_DROP:
1475                         flow_verbs_translate_action_drop(dev_flow, actions);
1476                         action_flags |= MLX5_FLOW_ACTION_DROP;
1477                         break;
1478                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1479                         flow_verbs_translate_action_queue(dev_flow, actions);
1480                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1481                         break;
1482                 case RTE_FLOW_ACTION_TYPE_RSS:
1483                         flow_verbs_translate_action_rss(dev_flow, actions);
1484                         action_flags |= MLX5_FLOW_ACTION_RSS;
1485                         break;
1486                 case RTE_FLOW_ACTION_TYPE_COUNT:
1487                         ret = flow_verbs_translate_action_count(dev_flow,
1488                                                                 actions,
1489                                                                 dev, error);
1490                         if (ret < 0)
1491                                 return ret;
1492                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1493                         break;
1494                 default:
1495                         return rte_flow_error_set(error, ENOTSUP,
1496                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1497                                                   actions,
1498                                                   "action not supported");
1499                 }
1500         }
1501         dev_flow->actions = action_flags;
1502         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1503                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1504
1505                 switch (items->type) {
1506                 case RTE_FLOW_ITEM_TYPE_VOID:
1507                         break;
1508                 case RTE_FLOW_ITEM_TYPE_ETH:
1509                         flow_verbs_translate_item_eth(dev_flow, items,
1510                                                       item_flags);
1511                         subpriority = MLX5_PRIORITY_MAP_L2;
1512                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1513                                                MLX5_FLOW_LAYER_OUTER_L2;
1514                         break;
1515                 case RTE_FLOW_ITEM_TYPE_VLAN:
1516                         flow_verbs_translate_item_vlan(dev_flow, items,
1517                                                        item_flags);
1518                         subpriority = MLX5_PRIORITY_MAP_L2;
1519                         item_flags |= tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
1520                                                 MLX5_FLOW_LAYER_INNER_VLAN) :
1521                                                (MLX5_FLOW_LAYER_OUTER_L2 |
1522                                                 MLX5_FLOW_LAYER_OUTER_VLAN);
1523                         break;
1524                 case RTE_FLOW_ITEM_TYPE_IPV4:
1525                         flow_verbs_translate_item_ipv4(dev_flow, items,
1526                                                        item_flags);
1527                         subpriority = MLX5_PRIORITY_MAP_L3;
1528                         dev_flow->hash_fields |=
1529                                 mlx5_flow_hashfields_adjust
1530                                         (dev_flow, tunnel,
1531                                          MLX5_IPV4_LAYER_TYPES,
1532                                          MLX5_IPV4_IBV_RX_HASH);
1533                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1534                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1535                         break;
1536                 case RTE_FLOW_ITEM_TYPE_IPV6:
1537                         flow_verbs_translate_item_ipv6(dev_flow, items,
1538                                                        item_flags);
1539                         subpriority = MLX5_PRIORITY_MAP_L3;
1540                         dev_flow->hash_fields |=
1541                                 mlx5_flow_hashfields_adjust
1542                                         (dev_flow, tunnel,
1543                                          MLX5_IPV6_LAYER_TYPES,
1544                                          MLX5_IPV6_IBV_RX_HASH);
1545                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1546                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1547                         break;
1548                 case RTE_FLOW_ITEM_TYPE_TCP:
1549                         flow_verbs_translate_item_tcp(dev_flow, items,
1550                                                       item_flags);
1551                         subpriority = MLX5_PRIORITY_MAP_L4;
1552                         dev_flow->hash_fields |=
1553                                 mlx5_flow_hashfields_adjust
1554                                         (dev_flow, tunnel, ETH_RSS_TCP,
1555                                          (IBV_RX_HASH_SRC_PORT_TCP |
1556                                           IBV_RX_HASH_DST_PORT_TCP));
1557                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1558                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1559                         break;
1560                 case RTE_FLOW_ITEM_TYPE_UDP:
1561                         flow_verbs_translate_item_udp(dev_flow, items,
1562                                                       item_flags);
1563                         subpriority = MLX5_PRIORITY_MAP_L4;
1564                         dev_flow->hash_fields |=
1565                                 mlx5_flow_hashfields_adjust
1566                                         (dev_flow, tunnel, ETH_RSS_UDP,
1567                                          (IBV_RX_HASH_SRC_PORT_UDP |
1568                                           IBV_RX_HASH_DST_PORT_UDP));
1569                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1570                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1571                         break;
1572                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1573                         flow_verbs_translate_item_vxlan(dev_flow, items,
1574                                                         item_flags);
1575                         subpriority = MLX5_PRIORITY_MAP_L2;
1576                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1577                         break;
1578                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1579                         flow_verbs_translate_item_vxlan_gpe(dev_flow, items,
1580                                                             item_flags);
1581                         subpriority = MLX5_PRIORITY_MAP_L2;
1582                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1583                         break;
1584                 case RTE_FLOW_ITEM_TYPE_GRE:
1585                         flow_verbs_translate_item_gre(dev_flow, items,
1586                                                       item_flags);
1587                         subpriority = MLX5_PRIORITY_MAP_L2;
1588                         item_flags |= MLX5_FLOW_LAYER_GRE;
1589                         break;
1590                 case RTE_FLOW_ITEM_TYPE_MPLS:
1591                         flow_verbs_translate_item_mpls(dev_flow, items,
1592                                                        item_flags);
1593                         subpriority = MLX5_PRIORITY_MAP_L2;
1594                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1595                         break;
1596                 default:
1597                         return rte_flow_error_set(error, ENOTSUP,
1598                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1599                                                   NULL,
1600                                                   "item not supported");
1601                 }
1602         }
1603         dev_flow->layers = item_flags;
1604         dev_flow->verbs.attr->priority =
1605                 mlx5_flow_adjust_priority(dev, priority, subpriority);
1606         dev_flow->verbs.attr->port = (uint8_t)priv->ibv_port;
1607         return 0;
1608 }
1609
1610 /**
1611  * Remove the flow from the NIC but keeps it in memory.
1612  *
1613  * @param[in] dev
1614  *   Pointer to the Ethernet device structure.
1615  * @param[in, out] flow
1616  *   Pointer to flow structure.
1617  */
1618 static void
1619 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1620 {
1621         struct mlx5_flow_verbs *verbs;
1622         struct mlx5_flow *dev_flow;
1623
1624         if (!flow)
1625                 return;
1626         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1627                 verbs = &dev_flow->verbs;
1628                 if (verbs->flow) {
1629                         claim_zero(mlx5_glue->destroy_flow(verbs->flow));
1630                         verbs->flow = NULL;
1631                 }
1632                 if (verbs->hrxq) {
1633                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
1634                                 mlx5_hrxq_drop_release(dev);
1635                         else
1636                                 mlx5_hrxq_release(dev, verbs->hrxq);
1637                         verbs->hrxq = NULL;
1638                 }
1639                 if (dev_flow->verbs.vf_vlan.tag &&
1640                     dev_flow->verbs.vf_vlan.created) {
1641                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1642                 }
1643         }
1644 }
1645
1646 /**
1647  * Remove the flow from the NIC and the memory.
1648  *
1649  * @param[in] dev
1650  *   Pointer to the Ethernet device structure.
1651  * @param[in, out] flow
1652  *   Pointer to flow structure.
1653  */
1654 static void
1655 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1656 {
1657         struct mlx5_flow *dev_flow;
1658
1659         if (!flow)
1660                 return;
1661         flow_verbs_remove(dev, flow);
1662         while (!LIST_EMPTY(&flow->dev_flows)) {
1663                 dev_flow = LIST_FIRST(&flow->dev_flows);
1664                 LIST_REMOVE(dev_flow, next);
1665                 rte_free(dev_flow);
1666         }
1667         if (flow->counter) {
1668                 flow_verbs_counter_release(dev, flow->counter);
1669                 flow->counter = NULL;
1670         }
1671 }
1672
1673 /**
1674  * Apply the flow to the NIC.
1675  *
1676  * @param[in] dev
1677  *   Pointer to the Ethernet device structure.
1678  * @param[in, out] flow
1679  *   Pointer to flow structure.
1680  * @param[out] error
1681  *   Pointer to error structure.
1682  *
1683  * @return
1684  *   0 on success, a negative errno value otherwise and rte_errno is set.
1685  */
1686 static int
1687 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1688                  struct rte_flow_error *error)
1689 {
1690         struct mlx5_priv *priv = dev->data->dev_private;
1691         struct mlx5_flow_verbs *verbs;
1692         struct mlx5_flow *dev_flow;
1693         int err;
1694
1695         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1696                 verbs = &dev_flow->verbs;
1697                 if (dev_flow->actions & MLX5_FLOW_ACTION_DROP) {
1698                         verbs->hrxq = mlx5_hrxq_drop_new(dev);
1699                         if (!verbs->hrxq) {
1700                                 rte_flow_error_set
1701                                         (error, errno,
1702                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1703                                          "cannot get drop hash queue");
1704                                 goto error;
1705                         }
1706                 } else {
1707                         struct mlx5_hrxq *hrxq;
1708
1709                         MLX5_ASSERT(flow->rss.queue);
1710                         hrxq = mlx5_hrxq_get(dev, flow->rss.key,
1711                                              MLX5_RSS_HASH_KEY_LEN,
1712                                              dev_flow->hash_fields,
1713                                              (*flow->rss.queue),
1714                                              flow->rss.queue_num);
1715                         if (!hrxq)
1716                                 hrxq = mlx5_hrxq_new(dev, flow->rss.key,
1717                                                      MLX5_RSS_HASH_KEY_LEN,
1718                                                      dev_flow->hash_fields,
1719                                                      (*flow->rss.queue),
1720                                                      flow->rss.queue_num,
1721                                                      !!(dev_flow->layers &
1722                                                        MLX5_FLOW_LAYER_TUNNEL));
1723                         if (!hrxq) {
1724                                 rte_flow_error_set
1725                                         (error, rte_errno,
1726                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1727                                          "cannot get hash queue");
1728                                 goto error;
1729                         }
1730                         verbs->hrxq = hrxq;
1731                 }
1732                 verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp,
1733                                                      verbs->attr);
1734                 if (!verbs->flow) {
1735                         rte_flow_error_set(error, errno,
1736                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1737                                            NULL,
1738                                            "hardware refuses to create flow");
1739                         goto error;
1740                 }
1741                 if (priv->vmwa_context &&
1742                     dev_flow->verbs.vf_vlan.tag &&
1743                     !dev_flow->verbs.vf_vlan.created) {
1744                         /*
1745                          * The rule contains the VLAN pattern.
1746                          * For VF we are going to create VLAN
1747                          * interface to make hypervisor set correct
1748                          * e-Switch vport context.
1749                          */
1750                         mlx5_vlan_vmwa_acquire(dev, &dev_flow->verbs.vf_vlan);
1751                 }
1752         }
1753         return 0;
1754 error:
1755         err = rte_errno; /* Save rte_errno before cleanup. */
1756         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1757                 verbs = &dev_flow->verbs;
1758                 if (verbs->hrxq) {
1759                         if (dev_flow->actions & MLX5_FLOW_ACTION_DROP)
1760                                 mlx5_hrxq_drop_release(dev);
1761                         else
1762                                 mlx5_hrxq_release(dev, verbs->hrxq);
1763                         verbs->hrxq = NULL;
1764                 }
1765                 if (dev_flow->verbs.vf_vlan.tag &&
1766                     dev_flow->verbs.vf_vlan.created) {
1767                         mlx5_vlan_vmwa_release(dev, &dev_flow->verbs.vf_vlan);
1768                 }
1769         }
1770         rte_errno = err; /* Restore rte_errno. */
1771         return -rte_errno;
1772 }
1773
1774 /**
1775  * Query a flow.
1776  *
1777  * @see rte_flow_query()
1778  * @see rte_flow_ops
1779  */
1780 static int
1781 flow_verbs_query(struct rte_eth_dev *dev,
1782                  struct rte_flow *flow,
1783                  const struct rte_flow_action *actions,
1784                  void *data,
1785                  struct rte_flow_error *error)
1786 {
1787         int ret = -EINVAL;
1788
1789         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1790                 switch (actions->type) {
1791                 case RTE_FLOW_ACTION_TYPE_VOID:
1792                         break;
1793                 case RTE_FLOW_ACTION_TYPE_COUNT:
1794                         ret = flow_verbs_counter_query(dev, flow, data, error);
1795                         break;
1796                 default:
1797                         return rte_flow_error_set(error, ENOTSUP,
1798                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1799                                                   actions,
1800                                                   "action not supported");
1801                 }
1802         }
1803         return ret;
1804 }
1805
1806 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1807         .validate = flow_verbs_validate,
1808         .prepare = flow_verbs_prepare,
1809         .translate = flow_verbs_translate,
1810         .apply = flow_verbs_apply,
1811         .remove = flow_verbs_remove,
1812         .destroy = flow_verbs_destroy,
1813         .query = flow_verbs_query,
1814 };