drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_eth_ctrl.h>
  24 #include <rte_ethdev_driver.h>
  25 #include <rte_flow.h>
  26 #include <rte_flow_driver.h>
  27 #include <rte_malloc.h>
  28 #include <rte_ip.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35
  36 /**
  37  * Create Verbs flow counter with Verbs library.
  38  *
  39  * @param[in] dev
  40  *   Pointer to the Ethernet device structure.
  41  * @param[in, out] counter
  42  *   mlx5 flow counter object, contains the counter id,
  43  *   handle of created Verbs flow counter is returned
  44  *   in cs field (if counters are supported).
  45  *
  46  * @return
  47  *   0 On success else a negative errno value is returned
  48  *   and rte_errno is set.
  49  */
  50 static int
  51 flow_verbs_counter_create(struct rte_eth_dev *dev,
  52                           struct mlx5_flow_counter *counter)
  53 {
  54 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  55         struct priv *priv = dev->data->dev_private;
  56         struct ibv_counter_set_init_attr init = {
  57                          .counter_set_id = counter->id};
  58
  59         counter->cs = mlx5_glue->create_counter_set(priv->ctx, &init);
  60         if (!counter->cs) {
  61                 rte_errno = ENOTSUP;
  62                 return -ENOTSUP;
  63         }
  64         return 0;
  65 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
  66         struct priv *priv = dev->data->dev_private;
  67         struct ibv_counters_init_attr init = {0};
  68         struct ibv_counter_attach_attr attach = {0};
  69         int ret;
  70
  71         counter->cs = mlx5_glue->create_counters(priv->ctx, &init);
  72         if (!counter->cs) {
  73                 rte_errno = ENOTSUP;
  74                 return -ENOTSUP;
  75         }
  76         attach.counter_desc = IBV_COUNTER_PACKETS;
  77         attach.index = 0;
  78         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
  79         if (!ret) {
  80                 attach.counter_desc = IBV_COUNTER_BYTES;
  81                 attach.index = 1;
  82                 ret = mlx5_glue->attach_counters
  83                                         (counter->cs, &attach, NULL);
  84         }
  85         if (ret) {
  86                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
  87                 counter->cs = NULL;
  88                 rte_errno = ret;
  89                 return -ret;
  90         }
  91         return 0;
  92 #else
  93         (void)dev;
  94         (void)counter;
  95         rte_errno = ENOTSUP;
  96         return -ENOTSUP;
  97 #endif
  98 }
  99
 100 /**
 101  * Get a flow counter.
 102  *
 103  * @param[in] dev
 104  *   Pointer to the Ethernet device structure.
 105  * @param[in] shared
 106  *   Indicate if this counter is shared with other flows.
 107  * @param[in] id
 108  *   Counter identifier.
 109  *
 110  * @return
 111  *   A pointer to the counter, NULL otherwise and rte_errno is set.
 112  */
 113 static struct mlx5_flow_counter *
 114 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 115 {
 116         struct priv *priv = dev->data->dev_private;
 117         struct mlx5_flow_counter *cnt;
 118         int ret;
 119
 120         LIST_FOREACH(cnt, &priv->flow_counters, next) {
 121                 if (!cnt->shared || cnt->shared != shared)
 122                         continue;
 123                 if (cnt->id != id)
 124                         continue;
 125                 cnt->ref_cnt++;
 126                 return cnt;
 127         }
 128         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 129         if (!cnt) {
 130                 rte_errno = ENOMEM;
 131                 return NULL;
 132         }
 133         cnt->id = id;
 134         cnt->shared = shared;
 135         cnt->ref_cnt = 1;
 136         cnt->hits = 0;
 137         cnt->bytes = 0;
 138         /* Create counter with Verbs. */
 139         ret = flow_verbs_counter_create(dev, cnt);
 140         if (!ret) {
 141                 LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
 142                 return cnt;
 143         }
 144         /* Some error occurred in Verbs library. */
 145         rte_free(cnt);
 146         rte_errno = -ret;
 147         return NULL;
 148 }
 149
 150 /**
 151  * Release a flow counter.
 152  *
 153  * @param[in] counter
 154  *   Pointer to the counter handler.
 155  */
 156 static void
 157 flow_verbs_counter_release(struct mlx5_flow_counter *counter)
 158 {
 159         if (--counter->ref_cnt == 0) {
 160 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 161                 claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
 162 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 163                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 164 #endif
 165                 LIST_REMOVE(counter, next);
 166                 rte_free(counter);
 167         }
 168 }
 169
 170 /**
 171  * Query a flow counter via Verbs library call.
 172  *
 173  * @see rte_flow_query()
 174  * @see rte_flow_ops
 175  */
 176 static int
 177 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 178                          struct rte_flow *flow, void *data,
 179                          struct rte_flow_error *error)
 180 {
 181 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 182         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 183         if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 184                 struct rte_flow_query_count *qc = data;
 185                 uint64_t counters[2] = {0, 0};
 186 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 187                 struct ibv_query_counter_set_attr query_cs_attr = {
 188                         .cs = flow->counter->cs,
 189                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 190                 };
 191                 struct ibv_counter_set_data query_out = {
 192                         .out = counters,
 193                         .outlen = 2 * sizeof(uint64_t),
 194                 };
 195                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 196                                                        &query_out);
 197 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 198                 int err = mlx5_glue->query_counters
 199                                (flow->counter->cs, counters,
 200                                 RTE_DIM(counters),
 201                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 202 #endif
 203                 if (err)
 204                         return rte_flow_error_set
 205                                 (error, err,
 206                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 207                                  NULL,
 208                                  "cannot read counter");
 209                 qc->hits_set = 1;
 210                 qc->bytes_set = 1;
 211                 qc->hits = counters[0] - flow->counter->hits;
 212                 qc->bytes = counters[1] - flow->counter->bytes;
 213                 if (qc->reset) {
 214                         flow->counter->hits = counters[0];
 215                         flow->counter->bytes = counters[1];
 216                 }
 217                 return 0;
 218         }
 219         return rte_flow_error_set(error, EINVAL,
 220                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 221                                   NULL,
 222                                   "flow does not have counter");
 223 #else
 224         (void)flow;
 225         (void)data;
 226         return rte_flow_error_set(error, ENOTSUP,
 227                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 228                                   NULL,
 229                                   "counters are not available");
 230 #endif
 231 }
 232
 233 /**
 234  * Add a verbs item specification into @p flow.
 235  *
 236  * @param[in, out] flow
 237  *   Pointer to flow structure.
 238  * @param[in] src
 239  *   Create specification.
 240  * @param[in] size
 241  *   Size in bytes of the specification to copy.
 242  */
 243 static void
 244 flow_verbs_spec_add(struct mlx5_flow *flow, void *src, unsigned int size)
 245 {
 246         struct mlx5_flow_verbs *verbs = &flow->verbs;
 247
 248         if (verbs->specs) {
 249                 void *dst;
 250
 251                 dst = (void *)(verbs->specs + verbs->size);
 252                 memcpy(dst, src, size);
 253                 ++verbs->attr->num_of_specs;
 254         }
 255         verbs->size += size;
 256 }
 257
 258 /**
 259  * Convert the @p item into a Verbs specification. This function assumes that
 260  * the input is valid and that there is space to insert the requested item
 261  * into the flow.
 262  *
 263  * @param[in] item
 264  *   Item specification.
 265  * @param[in] item_flags
 266  *   Bit field with all detected items.
 267  * @param[in, out] dev_flow
 268  *   Pointer to dev_flow structure.
 269  */
 270 static void
 271 flow_verbs_translate_item_eth(const struct rte_flow_item *item,
 272                               uint64_t *item_flags,
 273                               struct mlx5_flow *dev_flow)
 274 {
 275         const struct rte_flow_item_eth *spec = item->spec;
 276         const struct rte_flow_item_eth *mask = item->mask;
 277         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 278         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 279         struct ibv_flow_spec_eth eth = {
 280                 .type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 281                 .size = size,
 282         };
 283
 284         if (!mask)
 285                 mask = &rte_flow_item_eth_mask;
 286         if (spec) {
 287                 unsigned int i;
 288
 289                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 290                 memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 291                 eth.val.ether_type = spec->type;
 292                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 293                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 294                 eth.mask.ether_type = mask->type;
 295                 /* Remove unwanted bits from values. */
 296                 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 297                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 298                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 299                 }
 300                 eth.val.ether_type &= eth.mask.ether_type;
 301                 dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 302         }
 303         flow_verbs_spec_add(dev_flow, &eth, size);
 304         *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 305                                 MLX5_FLOW_LAYER_OUTER_L2;
 306 }
 307
 308 /**
 309  * Update the VLAN tag in the Verbs Ethernet specification.
 310  * This function assumes that the input is valid and there is space to add
 311  * the requested item.
 312  *
 313  * @param[in, out] attr
 314  *   Pointer to Verbs attributes structure.
 315  * @param[in] eth
 316  *   Verbs structure containing the VLAN information to copy.
 317  */
 318 static void
 319 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 320                             struct ibv_flow_spec_eth *eth)
 321 {
 322         unsigned int i;
 323         const enum ibv_flow_spec_type search = eth->type;
 324         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 325                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 326
 327         for (i = 0; i != attr->num_of_specs; ++i) {
 328                 if (hdr->type == search) {
 329                         struct ibv_flow_spec_eth *e =
 330                                 (struct ibv_flow_spec_eth *)hdr;
 331
 332                         e->val.vlan_tag = eth->val.vlan_tag;
 333                         e->mask.vlan_tag = eth->mask.vlan_tag;
 334                         e->val.ether_type = eth->val.ether_type;
 335                         e->mask.ether_type = eth->mask.ether_type;
 336                         break;
 337                 }
 338                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 339         }
 340 }
 341
 342 /**
 343  * Convert the @p item into a Verbs specification. This function assumes that
 344  * the input is valid and that there is space to insert the requested item
 345  * into the flow.
 346  *
 347  * @param[in] item
 348  *   Item specification.
 349  * @param[in, out] item_flags
 350  *   Bit mask that holds all detected items.
 351  * @param[in, out] dev_flow
 352  *   Pointer to dev_flow structure.
 353  */
 354 static void
 355 flow_verbs_translate_item_vlan(const struct rte_flow_item *item,
 356                                uint64_t *item_flags,
 357                                struct mlx5_flow *dev_flow)
 358 {
 359         const struct rte_flow_item_vlan *spec = item->spec;
 360         const struct rte_flow_item_vlan *mask = item->mask;
 361         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 362         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 363         struct ibv_flow_spec_eth eth = {
 364                 .type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 365                 .size = size,
 366         };
 367         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 368                                       MLX5_FLOW_LAYER_OUTER_L2;
 369
 370         if (!mask)
 371                 mask = &rte_flow_item_vlan_mask;
 372         if (spec) {
 373                 eth.val.vlan_tag = spec->tci;
 374                 eth.mask.vlan_tag = mask->tci;
 375                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 376                 eth.val.ether_type = spec->inner_type;
 377                 eth.mask.ether_type = mask->inner_type;
 378                 eth.val.ether_type &= eth.mask.ether_type;
 379         }
 380         if (!(*item_flags & l2m)) {
 381                 dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 382                 flow_verbs_spec_add(dev_flow, &eth, size);
 383         } else {
 384                 flow_verbs_item_vlan_update(dev_flow->verbs.attr, &eth);
 385                 size = 0; /* Only an update is done in eth specification. */
 386         }
 387         *item_flags |= tunnel ?
 388                        (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_VLAN) :
 389                        (MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN);
 390 }
 391
 392 /**
 393  * Convert the @p item into a Verbs specification. This function assumes that
 394  * the input is valid and that there is space to insert the requested item
 395  * into the flow.
 396  *
 397  * @param[in] item
 398  *   Item specification.
 399  * @param[in, out] item_flags
 400  *   Bit mask that marks all detected items.
 401  * @param[in, out] dev_flow
 402  *   Pointer to sepacific flow structure.
 403  */
 404 static void
 405 flow_verbs_translate_item_ipv4(const struct rte_flow_item *item,
 406                                uint64_t *item_flags,
 407                                struct mlx5_flow *dev_flow)
 408 {
 409         const struct rte_flow_item_ipv4 *spec = item->spec;
 410         const struct rte_flow_item_ipv4 *mask = item->mask;
 411         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 412         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 413         struct ibv_flow_spec_ipv4_ext ipv4 = {
 414                 .type = IBV_FLOW_SPEC_IPV4_EXT |
 415                         (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 416                 .size = size,
 417         };
 418
 419         if (!mask)
 420                 mask = &rte_flow_item_ipv4_mask;
 421         *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
 422                                 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 423         if (spec) {
 424                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 425                         .src_ip = spec->hdr.src_addr,
 426                         .dst_ip = spec->hdr.dst_addr,
 427                         .proto = spec->hdr.next_proto_id,
 428                         .tos = spec->hdr.type_of_service,
 429                 };
 430                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 431                         .src_ip = mask->hdr.src_addr,
 432                         .dst_ip = mask->hdr.dst_addr,
 433                         .proto = mask->hdr.next_proto_id,
 434                         .tos = mask->hdr.type_of_service,
 435                 };
 436                 /* Remove unwanted bits from values. */
 437                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 438                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 439                 ipv4.val.proto &= ipv4.mask.proto;
 440                 ipv4.val.tos &= ipv4.mask.tos;
 441         }
 442         dev_flow->verbs.hash_fields |=
 443                 mlx5_flow_hashfields_adjust(dev_flow, tunnel,
 444                                             MLX5_IPV4_LAYER_TYPES,
 445                                             MLX5_IPV4_IBV_RX_HASH);
 446         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L3;
 447         flow_verbs_spec_add(dev_flow, &ipv4, size);
 448 }
 449
 450 /**
 451  * Convert the @p item into a Verbs specification. This function assumes that
 452  * the input is valid and that there is space to insert the requested item
 453  * into the flow.
 454  *
 455  * @param[in] item
 456  *   Item specification.
 457  * @param[in, out] item_flags
 458  *   Bit mask that marks all detected items.
 459  * @param[in, out] dev_flow
 460  *   Pointer to sepacific flow structure.
 461  */
 462 static void
 463 flow_verbs_translate_item_ipv6(const struct rte_flow_item *item,
 464                                uint64_t *item_flags,
 465                                struct mlx5_flow *dev_flow)
 466 {
 467         const struct rte_flow_item_ipv6 *spec = item->spec;
 468         const struct rte_flow_item_ipv6 *mask = item->mask;
 469         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 470         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 471         struct ibv_flow_spec_ipv6 ipv6 = {
 472                 .type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 473                 .size = size,
 474         };
 475
 476         if (!mask)
 477                 mask = &rte_flow_item_ipv6_mask;
 478          *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
 479                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 480         if (spec) {
 481                 unsigned int i;
 482                 uint32_t vtc_flow_val;
 483                 uint32_t vtc_flow_mask;
 484
 485                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 486                        RTE_DIM(ipv6.val.src_ip));
 487                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 488                        RTE_DIM(ipv6.val.dst_ip));
 489                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 490                        RTE_DIM(ipv6.mask.src_ip));
 491                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 492                        RTE_DIM(ipv6.mask.dst_ip));
 493                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 494                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 495                 ipv6.val.flow_label =
 496                         rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
 497                                          IPV6_HDR_FL_SHIFT);
 498                 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
 499                                          IPV6_HDR_TC_SHIFT;
 500                 ipv6.val.next_hdr = spec->hdr.proto;
 501                 ipv6.val.hop_limit = spec->hdr.hop_limits;
 502                 ipv6.mask.flow_label =
 503                         rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
 504                                          IPV6_HDR_FL_SHIFT);
 505                 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
 506                                           IPV6_HDR_TC_SHIFT;
 507                 ipv6.mask.next_hdr = mask->hdr.proto;
 508                 ipv6.mask.hop_limit = mask->hdr.hop_limits;
 509                 /* Remove unwanted bits from values. */
 510                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 511                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 512                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 513                 }
 514                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 515                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 516                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 517                 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 518         }
 519         dev_flow->verbs.hash_fields |=
 520                 mlx5_flow_hashfields_adjust(dev_flow, tunnel,
 521                                             MLX5_IPV6_LAYER_TYPES,
 522                                             MLX5_IPV6_IBV_RX_HASH);
 523         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L3;
 524         flow_verbs_spec_add(dev_flow, &ipv6, size);
 525 }
 526
 527 /**
 528  * Convert the @p item into a Verbs specification. This function assumes that
 529  * the input is valid and that there is space to insert the requested item
 530  * into the flow.
 531  *
 532  * @param[in] item
 533  *   Item specification.
 534  * @param[in, out] item_flags
 535  *   Bit mask that marks all detected items.
 536  * @param[in, out] dev_flow
 537  *   Pointer to sepacific flow structure.
 538  */
 539 static void
 540 flow_verbs_translate_item_udp(const struct rte_flow_item *item,
 541                               uint64_t *item_flags,
 542                               struct mlx5_flow *dev_flow)
 543 {
 544         const struct rte_flow_item_udp *spec = item->spec;
 545         const struct rte_flow_item_udp *mask = item->mask;
 546         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 547         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 548         struct ibv_flow_spec_tcp_udp udp = {
 549                 .type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 550                 .size = size,
 551         };
 552
 553         if (!mask)
 554                 mask = &rte_flow_item_udp_mask;
 555         *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
 556                                 MLX5_FLOW_LAYER_OUTER_L4_UDP;
 557         if (spec) {
 558                 udp.val.dst_port = spec->hdr.dst_port;
 559                 udp.val.src_port = spec->hdr.src_port;
 560                 udp.mask.dst_port = mask->hdr.dst_port;
 561                 udp.mask.src_port = mask->hdr.src_port;
 562                 /* Remove unwanted bits from values. */
 563                 udp.val.src_port &= udp.mask.src_port;
 564                 udp.val.dst_port &= udp.mask.dst_port;
 565         }
 566         dev_flow->verbs.hash_fields |=
 567                 mlx5_flow_hashfields_adjust(dev_flow, tunnel, ETH_RSS_UDP,
 568                                             (IBV_RX_HASH_SRC_PORT_UDP |
 569                                              IBV_RX_HASH_DST_PORT_UDP));
 570         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L4;
 571         flow_verbs_spec_add(dev_flow, &udp, size);
 572 }
 573
 574 /**
 575  * Convert the @p item into a Verbs specification. This function assumes that
 576  * the input is valid and that there is space to insert the requested item
 577  * into the flow.
 578  *
 579  * @param[in] item
 580  *   Item specification.
 581  * @param[in, out] item_flags
 582  *   Bit mask that marks all detected items.
 583  * @param[in, out] dev_flow
 584  *   Pointer to sepacific flow structure.
 585  */
 586 static void
 587 flow_verbs_translate_item_tcp(const struct rte_flow_item *item,
 588                               uint64_t *item_flags,
 589                               struct mlx5_flow *dev_flow)
 590 {
 591         const struct rte_flow_item_tcp *spec = item->spec;
 592         const struct rte_flow_item_tcp *mask = item->mask;
 593         const int tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 594         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 595         struct ibv_flow_spec_tcp_udp tcp = {
 596                 .type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 597                 .size = size,
 598         };
 599
 600         if (!mask)
 601                 mask = &rte_flow_item_tcp_mask;
 602         *item_flags |=  tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
 603                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
 604         if (spec) {
 605                 tcp.val.dst_port = spec->hdr.dst_port;
 606                 tcp.val.src_port = spec->hdr.src_port;
 607                 tcp.mask.dst_port = mask->hdr.dst_port;
 608                 tcp.mask.src_port = mask->hdr.src_port;
 609                 /* Remove unwanted bits from values. */
 610                 tcp.val.src_port &= tcp.mask.src_port;
 611                 tcp.val.dst_port &= tcp.mask.dst_port;
 612         }
 613         dev_flow->verbs.hash_fields |=
 614                 mlx5_flow_hashfields_adjust(dev_flow, tunnel, ETH_RSS_TCP,
 615                                             (IBV_RX_HASH_SRC_PORT_TCP |
 616                                              IBV_RX_HASH_DST_PORT_TCP));
 617         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L4;
 618         flow_verbs_spec_add(dev_flow, &tcp, size);
 619 }
 620
 621 /**
 622  * Convert the @p item into a Verbs specification. This function assumes that
 623  * the input is valid and that there is space to insert the requested item
 624  * into the flow.
 625  *
 626  * @param[in] item
 627  *   Item specification.
 628  * @param[in, out] item_flags
 629  *   Bit mask that marks all detected items.
 630  * @param[in, out] dev_flow
 631  *   Pointer to sepacific flow structure.
 632  */
 633 static void
 634 flow_verbs_translate_item_vxlan(const struct rte_flow_item *item,
 635                                 uint64_t *item_flags,
 636                                 struct mlx5_flow *dev_flow)
 637 {
 638         const struct rte_flow_item_vxlan *spec = item->spec;
 639         const struct rte_flow_item_vxlan *mask = item->mask;
 640         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 641         struct ibv_flow_spec_tunnel vxlan = {
 642                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 643                 .size = size,
 644         };
 645         union vni {
 646                 uint32_t vlan_id;
 647                 uint8_t vni[4];
 648         } id = { .vlan_id = 0, };
 649
 650         if (!mask)
 651                 mask = &rte_flow_item_vxlan_mask;
 652         if (spec) {
 653                 memcpy(&id.vni[1], spec->vni, 3);
 654                 vxlan.val.tunnel_id = id.vlan_id;
 655                 memcpy(&id.vni[1], mask->vni, 3);
 656                 vxlan.mask.tunnel_id = id.vlan_id;
 657                 /* Remove unwanted bits from values. */
 658                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 659         }
 660         flow_verbs_spec_add(dev_flow, &vxlan, size);
 661         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 662         *item_flags |= MLX5_FLOW_LAYER_VXLAN;
 663 }
 664
 665 /**
 666  * Convert the @p item into a Verbs specification. This function assumes that
 667  * the input is valid and that there is space to insert the requested item
 668  * into the flow.
 669  *
 670  * @param[in] item
 671  *   Item specification.
 672  * @param[in, out] item_flags
 673  *   Bit mask that marks all detected items.
 674  * @param[in, out] dev_flow
 675  *   Pointer to sepacific flow structure.
 676  */
 677 static void
 678 flow_verbs_translate_item_vxlan_gpe(const struct rte_flow_item *item,
 679                                     uint64_t *item_flags,
 680                                     struct mlx5_flow *dev_flow)
 681 {
 682         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 683         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 684         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 685         struct ibv_flow_spec_tunnel vxlan_gpe = {
 686                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 687                 .size = size,
 688         };
 689         union vni {
 690                 uint32_t vlan_id;
 691                 uint8_t vni[4];
 692         } id = { .vlan_id = 0, };
 693
 694         if (!mask)
 695                 mask = &rte_flow_item_vxlan_gpe_mask;
 696         if (spec) {
 697                 memcpy(&id.vni[1], spec->vni, 3);
 698                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 699                 memcpy(&id.vni[1], mask->vni, 3);
 700                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 701                 /* Remove unwanted bits from values. */
 702                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 703         }
 704         flow_verbs_spec_add(dev_flow, &vxlan_gpe, size);
 705         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 706         *item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
 707 }
 708
 709 /**
 710  * Update the protocol in Verbs IPv4/IPv6 spec.
 711  *
 712  * @param[in, out] attr
 713  *   Pointer to Verbs attributes structure.
 714  * @param[in] search
 715  *   Specification type to search in order to update the IP protocol.
 716  * @param[in] protocol
 717  *   Protocol value to set if none is present in the specification.
 718  */
 719 static void
 720 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 721                                        enum ibv_flow_spec_type search,
 722                                        uint8_t protocol)
 723 {
 724         unsigned int i;
 725         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 726                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 727
 728         if (!attr)
 729                 return;
 730         for (i = 0; i != attr->num_of_specs; ++i) {
 731                 if (hdr->type == search) {
 732                         union {
 733                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 734                                 struct ibv_flow_spec_ipv6 *ipv6;
 735                         } ip;
 736
 737                         switch (search) {
 738                         case IBV_FLOW_SPEC_IPV4_EXT:
 739                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 740                                 if (!ip.ipv4->val.proto) {
 741                                         ip.ipv4->val.proto = protocol;
 742                                         ip.ipv4->mask.proto = 0xff;
 743                                 }
 744                                 break;
 745                         case IBV_FLOW_SPEC_IPV6:
 746                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 747                                 if (!ip.ipv6->val.next_hdr) {
 748                                         ip.ipv6->val.next_hdr = protocol;
 749                                         ip.ipv6->mask.next_hdr = 0xff;
 750                                 }
 751                                 break;
 752                         default:
 753                                 break;
 754                         }
 755                         break;
 756                 }
 757                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 758         }
 759 }
 760
 761 /**
 762  * Convert the @p item into a Verbs specification. This function assumes that
 763  * the input is valid and that there is space to insert the requested item
 764  * into the flow.
 765  *
 766  * @param[in] item
 767  *   Item specification.
 768  * @param[in, out] item_flags
 769  *   Bit mask that marks all detected items.
 770  * @param[in, out] dev_flow
 771  *   Pointer to sepacific flow structure.
 772  */
 773 static void
 774 flow_verbs_translate_item_gre(const struct rte_flow_item *item __rte_unused,
 775                               uint64_t *item_flags,
 776                               struct mlx5_flow *dev_flow)
 777 {
 778         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
 779 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 780         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 781         struct ibv_flow_spec_tunnel tunnel = {
 782                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 783                 .size = size,
 784         };
 785 #else
 786         const struct rte_flow_item_gre *spec = item->spec;
 787         const struct rte_flow_item_gre *mask = item->mask;
 788         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 789         struct ibv_flow_spec_gre tunnel = {
 790                 .type = IBV_FLOW_SPEC_GRE,
 791                 .size = size,
 792         };
 793
 794         if (!mask)
 795                 mask = &rte_flow_item_gre_mask;
 796         if (spec) {
 797                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 798                 tunnel.val.protocol = spec->protocol;
 799                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 800                 tunnel.mask.protocol = mask->protocol;
 801                 /* Remove unwanted bits from values. */
 802                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 803                 tunnel.val.protocol &= tunnel.mask.protocol;
 804                 tunnel.val.key &= tunnel.mask.key;
 805         }
 806 #endif
 807         if (*item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 808                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 809                                                        IBV_FLOW_SPEC_IPV4_EXT,
 810                                                        IPPROTO_GRE);
 811         else
 812                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 813                                                        IBV_FLOW_SPEC_IPV6,
 814                                                        IPPROTO_GRE);
 815         flow_verbs_spec_add(dev_flow, &tunnel, size);
 816         verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
 817         *item_flags |= MLX5_FLOW_LAYER_GRE;
 818 }
 819
 820 /**
 821  * Convert the @p action into a Verbs specification. This function assumes that
 822  * the input is valid and that there is space to insert the requested action
 823  * into the flow. This function also return the action that was added.
 824  *
 825  * @param[in] item
 826  *   Item specification.
 827  * @param[in, out] item_flags
 828  *   Bit mask that marks all detected items.
 829  * @param[in, out] dev_flow
 830  *   Pointer to sepacific flow structure.
 831  */
 832 static void
 833 flow_verbs_translate_item_mpls(const struct rte_flow_item *item __rte_unused,
 834                                uint64_t *action_flags __rte_unused,
 835                                struct mlx5_flow *dev_flow __rte_unused)
 836 {
 837 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 838         const struct rte_flow_item_mpls *spec = item->spec;
 839         const struct rte_flow_item_mpls *mask = item->mask;
 840         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 841         struct ibv_flow_spec_mpls mpls = {
 842                 .type = IBV_FLOW_SPEC_MPLS,
 843                 .size = size,
 844         };
 845
 846         if (!mask)
 847                 mask = &rte_flow_item_mpls_mask;
 848         if (spec) {
 849                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 850                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 851                 /* Remove unwanted bits from values.  */
 852                 mpls.val.label &= mpls.mask.label;
 853         }
 854         flow_verbs_spec_add(dev_flow, &mpls, size);
 855         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 856         *action_flags |= MLX5_FLOW_LAYER_MPLS;
 857 #endif
 858 }
 859
 860 /**
 861  * Convert the @p action into a Verbs specification. This function assumes that
 862  * the input is valid and that there is space to insert the requested action
 863  * into the flow. This function also return the action that was added.
 864  *
 865  * @param[in, out] action_flags
 866  *   Pointer to the detected actions.
 867  * @param[in] dev_flow
 868  *   Pointer to mlx5_flow.
 869  */
 870 static void
 871 flow_verbs_translate_action_drop(uint64_t *action_flags,
 872                                  struct mlx5_flow *dev_flow)
 873 {
 874         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 875         struct ibv_flow_spec_action_drop drop = {
 876                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 877                         .size = size,
 878         };
 879
 880         flow_verbs_spec_add(dev_flow, &drop, size);
 881         *action_flags |= MLX5_FLOW_ACTION_DROP;
 882 }
 883
 884 /**
 885  * Convert the @p action into a Verbs specification. This function assumes that
 886  * the input is valid and that there is space to insert the requested action
 887  * into the flow. This function also return the action that was added.
 888  *
 889  * @param[in] action
 890  *   Action configuration.
 891  * @param[in, out] action_flags
 892  *   Pointer to the detected actions.
 893  * @param[in] dev_flow
 894  *   Pointer to mlx5_flow.
 895  */
 896 static void
 897 flow_verbs_translate_action_queue(const struct rte_flow_action *action,
 898                                   uint64_t *action_flags,
 899                                   struct mlx5_flow *dev_flow)
 900 {
 901         const struct rte_flow_action_queue *queue = action->conf;
 902         struct rte_flow *flow = dev_flow->flow;
 903
 904         if (flow->queue)
 905                 (*flow->queue)[0] = queue->index;
 906         flow->rss.queue_num = 1;
 907         *action_flags |= MLX5_FLOW_ACTION_QUEUE;
 908 }
 909
 910 /**
 911  * Convert the @p action into a Verbs specification. This function assumes that
 912  * the input is valid and that there is space to insert the requested action
 913  * into the flow. This function also return the action that was added.
 914  *
 915  * @param[in] action
 916  *   Action configuration.
 917  * @param[in, out] action_flags
 918  *   Pointer to the detected actions.
 919  * @param[in] dev_flow
 920  *   Pointer to mlx5_flow.
 921  */
 922 static void
 923 flow_verbs_translate_action_rss(const struct rte_flow_action *action,
 924                                 uint64_t *action_flags,
 925                                 struct mlx5_flow *dev_flow)
 926 {
 927         const struct rte_flow_action_rss *rss = action->conf;
 928         const uint8_t *rss_key;
 929         struct rte_flow *flow = dev_flow->flow;
 930
 931         if (flow->queue)
 932                 memcpy((*flow->queue), rss->queue,
 933                        rss->queue_num * sizeof(uint16_t));
 934         flow->rss.queue_num = rss->queue_num;
 935         /* NULL RSS key indicates default RSS key. */
 936         rss_key = !rss->key ? rss_hash_default_key : rss->key;
 937         memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
 938         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
 939         flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
 940         flow->rss.level = rss->level;
 941         *action_flags |= MLX5_FLOW_ACTION_RSS;
 942 }
 943
 944 /**
 945  * Convert the @p action into a Verbs specification. This function assumes that
 946  * the input is valid and that there is space to insert the requested action
 947  * into the flow. This function also return the action that was added.
 948  *
 949  * @param[in] action
 950  *   Action configuration.
 951  * @param[in, out] action_flags
 952  *   Pointer to the detected actions.
 953  * @param[in] dev_flow
 954  *   Pointer to mlx5_flow.
 955  */
 956 static void
 957 flow_verbs_translate_action_flag
 958                         (const struct rte_flow_action *action __rte_unused,
 959                          uint64_t *action_flags,
 960                          struct mlx5_flow *dev_flow)
 961 {
 962         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 963         struct ibv_flow_spec_action_tag tag = {
 964                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 965                 .size = size,
 966                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 967         };
 968         *action_flags |= MLX5_FLOW_ACTION_MARK;
 969         flow_verbs_spec_add(dev_flow, &tag, size);
 970 }
 971
 972 /**
 973  * Update verbs specification to modify the flag to mark.
 974  *
 975  * @param[in, out] verbs
 976  *   Pointer to the mlx5_flow_verbs structure.
 977  * @param[in] mark_id
 978  *   Mark identifier to replace the flag.
 979  */
 980 static void
 981 flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
 982 {
 983         struct ibv_spec_header *hdr;
 984         int i;
 985
 986         if (!verbs)
 987                 return;
 988         /* Update Verbs specification. */
 989         hdr = (struct ibv_spec_header *)verbs->specs;
 990         if (!hdr)
 991                 return;
 992         for (i = 0; i != verbs->attr->num_of_specs; ++i) {
 993                 if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
 994                         struct ibv_flow_spec_action_tag *t =
 995                                 (struct ibv_flow_spec_action_tag *)hdr;
 996
 997                         t->tag_id = mlx5_flow_mark_set(mark_id);
 998                 }
 999                 hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
1000         }
1001 }
1002
1003 /**
1004  * Convert the @p action into a Verbs specification. This function assumes that
1005  * the input is valid and that there is space to insert the requested action
1006  * into the flow. This function also return the action that was added.
1007  *
1008  * @param[in] action
1009  *   Action configuration.
1010  * @param[in, out] action_flags
1011  *   Pointer to the detected actions.
1012  * @param[in] dev_flow
1013  *   Pointer to mlx5_flow.
1014  */
1015 static void
1016 flow_verbs_translate_action_mark(const struct rte_flow_action *action,
1017                                  uint64_t *action_flags,
1018                                  struct mlx5_flow *dev_flow)
1019 {
1020         const struct rte_flow_action_mark *mark = action->conf;
1021         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1022         struct ibv_flow_spec_action_tag tag = {
1023                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1024                 .size = size,
1025         };
1026         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
1027
1028         if (*action_flags & MLX5_FLOW_ACTION_FLAG) {
1029                 flow_verbs_mark_update(verbs, mark->id);
1030                 size = 0;
1031         } else {
1032                 tag.tag_id = mlx5_flow_mark_set(mark->id);
1033                 flow_verbs_spec_add(dev_flow, &tag, size);
1034         }
1035         *action_flags |= MLX5_FLOW_ACTION_MARK;
1036 }
1037
1038 /**
1039  * Convert the @p action into a Verbs specification. This function assumes that
1040  * the input is valid and that there is space to insert the requested action
1041  * into the flow. This function also return the action that was added.
1042  *
1043  * @param[in] dev
1044  *   Pointer to the Ethernet device structure.
1045  * @param[in] action
1046  *   Action configuration.
1047  * @param[in, out] action_flags
1048  *   Pointer to the detected actions.
1049  * @param[in] dev_flow
1050  *   Pointer to mlx5_flow.
1051  * @param[out] error
1052  *   Pointer to error structure.
1053  *
1054  * @return
1055  *   0 On success else a negative errno value is returned and rte_errno is set.
1056  */
1057 static int
1058 flow_verbs_translate_action_count(struct rte_eth_dev *dev,
1059                                   const struct rte_flow_action *action,
1060                                   uint64_t *action_flags,
1061                                   struct mlx5_flow *dev_flow,
1062                                   struct rte_flow_error *error)
1063 {
1064         const struct rte_flow_action_count *count = action->conf;
1065         struct rte_flow *flow = dev_flow->flow;
1066 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1067         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1068         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1069         struct ibv_flow_spec_counter_action counter = {
1070                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1071                 .size = size,
1072         };
1073 #endif
1074
1075         if (!flow->counter) {
1076                 flow->counter = flow_verbs_counter_new(dev, count->shared,
1077                                                        count->id);
1078                 if (!flow->counter)
1079                         return rte_flow_error_set(error, rte_errno,
1080                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1081                                                   action,
1082                                                   "cannot get counter"
1083                                                   " context.");
1084         }
1085         *action_flags |= MLX5_FLOW_ACTION_COUNT;
1086 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1087         counter.counter_set_handle = flow->counter->cs->handle;
1088         flow_verbs_spec_add(dev_flow, &counter, size);
1089 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1090         counter.counters = flow->counter->cs;
1091         flow_verbs_spec_add(dev_flow, &counter, size);
1092 #endif
1093         return 0;
1094 }
1095
1096 /**
1097  * Internal validation function. For validating both actions and items.
1098  *
1099  * @param[in] dev
1100  *   Pointer to the Ethernet device structure.
1101  * @param[in] attr
1102  *   Pointer to the flow attributes.
1103  * @param[in] items
1104  *   Pointer to the list of items.
1105  * @param[in] actions
1106  *   Pointer to the list of actions.
1107  * @param[out] error
1108  *   Pointer to the error structure.
1109  *
1110  * @return
1111  *   0 on success, a negative errno value otherwise and rte_errno is set.
1112  */
1113 static int
1114 flow_verbs_validate(struct rte_eth_dev *dev,
1115                     const struct rte_flow_attr *attr,
1116                     const struct rte_flow_item items[],
1117                     const struct rte_flow_action actions[],
1118                     struct rte_flow_error *error)
1119 {
1120         int ret;
1121         uint64_t action_flags = 0;
1122         uint64_t item_flags = 0;
1123         int tunnel = 0;
1124         uint8_t next_protocol = 0xff;
1125
1126         if (items == NULL)
1127                 return -1;
1128         ret = mlx5_flow_validate_attributes(dev, attr, error);
1129         if (ret < 0)
1130                 return ret;
1131         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1132                 int ret = 0;
1133
1134                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1135                 switch (items->type) {
1136                 case RTE_FLOW_ITEM_TYPE_VOID:
1137                         break;
1138                 case RTE_FLOW_ITEM_TYPE_ETH:
1139                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1140                                                           error);
1141                         if (ret < 0)
1142                                 return ret;
1143                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1144                                                MLX5_FLOW_LAYER_OUTER_L2;
1145                         break;
1146                 case RTE_FLOW_ITEM_TYPE_VLAN:
1147                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1148                                                            error);
1149                         if (ret < 0)
1150                                 return ret;
1151                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1152                                                MLX5_FLOW_LAYER_OUTER_VLAN;
1153                         break;
1154                 case RTE_FLOW_ITEM_TYPE_IPV4:
1155                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1156                                                            error);
1157                         if (ret < 0)
1158                                 return ret;
1159                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1160                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1161                         if (items->mask != NULL &&
1162                             ((const struct rte_flow_item_ipv4 *)
1163                              items->mask)->hdr.next_proto_id)
1164                                 next_protocol =
1165                                         ((const struct rte_flow_item_ipv4 *)
1166                                          (items->spec))->hdr.next_proto_id;
1167                         break;
1168                 case RTE_FLOW_ITEM_TYPE_IPV6:
1169                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1170                                                            error);
1171                         if (ret < 0)
1172                                 return ret;
1173                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1174                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1175                         if (items->mask != NULL &&
1176                             ((const struct rte_flow_item_ipv6 *)
1177                              items->mask)->hdr.proto)
1178                                 next_protocol =
1179                                         ((const struct rte_flow_item_ipv6 *)
1180                                          items->spec)->hdr.proto;
1181                         break;
1182                 case RTE_FLOW_ITEM_TYPE_UDP:
1183                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1184                                                           next_protocol,
1185                                                           error);
1186                         if (ret < 0)
1187                                 return ret;
1188                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1189                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1190                         break;
1191                 case RTE_FLOW_ITEM_TYPE_TCP:
1192                         ret = mlx5_flow_validate_item_tcp
1193                                                 (items, item_flags,
1194                                                  next_protocol,
1195                                                  &rte_flow_item_tcp_mask,
1196                                                  error);
1197                         if (ret < 0)
1198                                 return ret;
1199                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1200                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1201                         break;
1202                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1203                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1204                                                             error);
1205                         if (ret < 0)
1206                                 return ret;
1207                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1208                         break;
1209                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1210                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1211                                                                 item_flags,
1212                                                                 dev, error);
1213                         if (ret < 0)
1214                                 return ret;
1215                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1216                         break;
1217                 case RTE_FLOW_ITEM_TYPE_GRE:
1218                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1219                                                           next_protocol, error);
1220                         if (ret < 0)
1221                                 return ret;
1222                         item_flags |= MLX5_FLOW_LAYER_GRE;
1223                         break;
1224                 case RTE_FLOW_ITEM_TYPE_MPLS:
1225                         ret = mlx5_flow_validate_item_mpls(items, item_flags,
1226                                                            next_protocol,
1227                                                            error);
1228                         if (ret < 0)
1229                                 return ret;
1230                         if (next_protocol != 0xff &&
1231                             next_protocol != IPPROTO_MPLS)
1232                                 return rte_flow_error_set
1233                                         (error, EINVAL,
1234                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1235                                          "protocol filtering not compatible"
1236                                          " with MPLS layer");
1237                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1238                         break;
1239                 default:
1240                         return rte_flow_error_set(error, ENOTSUP,
1241                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1242                                                   NULL, "item not supported");
1243                 }
1244         }
1245         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1246                 switch (actions->type) {
1247                 case RTE_FLOW_ACTION_TYPE_VOID:
1248                         break;
1249                 case RTE_FLOW_ACTION_TYPE_FLAG:
1250                         ret = mlx5_flow_validate_action_flag(action_flags,
1251                                                              attr,
1252                                                              error);
1253                         if (ret < 0)
1254                                 return ret;
1255                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1256                         break;
1257                 case RTE_FLOW_ACTION_TYPE_MARK:
1258                         ret = mlx5_flow_validate_action_mark(actions,
1259                                                              action_flags,
1260                                                              attr,
1261                                                              error);
1262                         if (ret < 0)
1263                                 return ret;
1264                         action_flags |= MLX5_FLOW_ACTION_MARK;
1265                         break;
1266                 case RTE_FLOW_ACTION_TYPE_DROP:
1267                         ret = mlx5_flow_validate_action_drop(action_flags,
1268                                                              attr,
1269                                                              error);
1270                         if (ret < 0)
1271                                 return ret;
1272                         action_flags |= MLX5_FLOW_ACTION_DROP;
1273                         break;
1274                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1275                         ret = mlx5_flow_validate_action_queue(actions,
1276                                                               action_flags, dev,
1277                                                               attr,
1278                                                               error);
1279                         if (ret < 0)
1280                                 return ret;
1281                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1282                         break;
1283                 case RTE_FLOW_ACTION_TYPE_RSS:
1284                         ret = mlx5_flow_validate_action_rss(actions,
1285                                                             action_flags, dev,
1286                                                             attr,
1287                                                             error);
1288                         if (ret < 0)
1289                                 return ret;
1290                         action_flags |= MLX5_FLOW_ACTION_RSS;
1291                         break;
1292                 case RTE_FLOW_ACTION_TYPE_COUNT:
1293                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1294                         if (ret < 0)
1295                                 return ret;
1296                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1297                         break;
1298                 default:
1299                         return rte_flow_error_set(error, ENOTSUP,
1300                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1301                                                   actions,
1302                                                   "action not supported");
1303                 }
1304         }
1305         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1306                 return rte_flow_error_set(error, EINVAL,
1307                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1308                                           "no fate action is found");
1309         return 0;
1310 }
1311
1312 /**
1313  * Calculate the required bytes that are needed for the action part of the verbs
1314  * flow, in addtion returns bit-fields with all the detected action, in order to
1315  * avoid another interation over the actions.
1316  *
1317  * @param[in] actions
1318  *   Pointer to the list of actions.
1319  * @param[out] action_flags
1320  *   Pointer to the detected actions.
1321  *
1322  * @return
1323  *   The size of the memory needed for all actions.
1324  */
1325 static int
1326 flow_verbs_get_actions_and_size(const struct rte_flow_action actions[],
1327                                 uint64_t *action_flags)
1328 {
1329         int size = 0;
1330         uint64_t detected_actions = 0;
1331
1332         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1333                 switch (actions->type) {
1334                 case RTE_FLOW_ACTION_TYPE_VOID:
1335                         break;
1336                 case RTE_FLOW_ACTION_TYPE_FLAG:
1337                         size += sizeof(struct ibv_flow_spec_action_tag);
1338                         detected_actions |= MLX5_FLOW_ACTION_FLAG;
1339                         break;
1340                 case RTE_FLOW_ACTION_TYPE_MARK:
1341                         size += sizeof(struct ibv_flow_spec_action_tag);
1342                         detected_actions |= MLX5_FLOW_ACTION_MARK;
1343                         break;
1344                 case RTE_FLOW_ACTION_TYPE_DROP:
1345                         size += sizeof(struct ibv_flow_spec_action_drop);
1346                         detected_actions |= MLX5_FLOW_ACTION_DROP;
1347                         break;
1348                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1349                         detected_actions |= MLX5_FLOW_ACTION_QUEUE;
1350                         break;
1351                 case RTE_FLOW_ACTION_TYPE_RSS:
1352                         detected_actions |= MLX5_FLOW_ACTION_RSS;
1353                         break;
1354                 case RTE_FLOW_ACTION_TYPE_COUNT:
1355 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1356         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1357                         size += sizeof(struct ibv_flow_spec_counter_action);
1358 #endif
1359                         detected_actions |= MLX5_FLOW_ACTION_COUNT;
1360                         break;
1361                 default:
1362                         break;
1363                 }
1364         }
1365         *action_flags = detected_actions;
1366         return size;
1367 }
1368
1369 /**
1370  * Calculate the required bytes that are needed for the item part of the verbs
1371  * flow, in addtion returns bit-fields with all the detected action, in order to
1372  * avoid another interation over the actions.
1373  *
1374  * @param[in] actions
1375  *   Pointer to the list of items.
1376  * @param[in, out] item_flags
1377  *   Pointer to the detected items.
1378  *
1379  * @return
1380  *   The size of the memory needed for all items.
1381  */
1382 static int
1383 flow_verbs_get_items_and_size(const struct rte_flow_item items[],
1384                               uint64_t *item_flags)
1385 {
1386         int size = 0;
1387         uint64_t detected_items = 0;
1388
1389         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1390                 int tunnel = !!(detected_items & MLX5_FLOW_LAYER_TUNNEL);
1391
1392                 switch (items->type) {
1393                 case RTE_FLOW_ITEM_TYPE_VOID:
1394                         break;
1395                 case RTE_FLOW_ITEM_TYPE_ETH:
1396                         size += sizeof(struct ibv_flow_spec_eth);
1397                         detected_items |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1398                                                    MLX5_FLOW_LAYER_OUTER_L2;
1399                         break;
1400                 case RTE_FLOW_ITEM_TYPE_VLAN:
1401                         size += sizeof(struct ibv_flow_spec_eth);
1402                         detected_items |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1403                                                    MLX5_FLOW_LAYER_OUTER_VLAN;
1404                         break;
1405                 case RTE_FLOW_ITEM_TYPE_IPV4:
1406                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1407                         detected_items |= tunnel ?
1408                                           MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1409                                           MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1410                         break;
1411                 case RTE_FLOW_ITEM_TYPE_IPV6:
1412                         size += sizeof(struct ibv_flow_spec_ipv6);
1413                         detected_items |= tunnel ?
1414                                           MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1415                                           MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1416                         break;
1417                 case RTE_FLOW_ITEM_TYPE_UDP:
1418                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1419                         detected_items |= tunnel ?
1420                                           MLX5_FLOW_LAYER_INNER_L4_UDP :
1421                                           MLX5_FLOW_LAYER_OUTER_L4_UDP;
1422                         break;
1423                 case RTE_FLOW_ITEM_TYPE_TCP:
1424                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1425                         detected_items |= tunnel ?
1426                                           MLX5_FLOW_LAYER_INNER_L4_TCP :
1427                                           MLX5_FLOW_LAYER_OUTER_L4_TCP;
1428                         break;
1429                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1430                         size += sizeof(struct ibv_flow_spec_tunnel);
1431                         detected_items |= MLX5_FLOW_LAYER_VXLAN;
1432                         break;
1433                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1434                         size += sizeof(struct ibv_flow_spec_tunnel);
1435                         detected_items |= MLX5_FLOW_LAYER_VXLAN_GPE;
1436                         break;
1437 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1438                 case RTE_FLOW_ITEM_TYPE_GRE:
1439                         size += sizeof(struct ibv_flow_spec_gre);
1440                         detected_items |= MLX5_FLOW_LAYER_GRE;
1441                         break;
1442                 case RTE_FLOW_ITEM_TYPE_MPLS:
1443                         size += sizeof(struct ibv_flow_spec_mpls);
1444                         detected_items |= MLX5_FLOW_LAYER_MPLS;
1445                         break;
1446 #else
1447                 case RTE_FLOW_ITEM_TYPE_GRE:
1448                         size += sizeof(struct ibv_flow_spec_tunnel);
1449                         detected_items |= MLX5_FLOW_LAYER_TUNNEL;
1450                         break;
1451 #endif
1452                 default:
1453                         break;
1454                 }
1455         }
1456         *item_flags = detected_items;
1457         return size;
1458 }
1459
1460 /**
1461  * Internal preparation function. Allocate mlx5_flow with the required size.
1462  * The required size is calculate based on the actions and items. This function
1463  * also returns the detected actions and items for later use.
1464  *
1465  * @param[in] attr
1466  *   Pointer to the flow attributes.
1467  * @param[in] items
1468  *   Pointer to the list of items.
1469  * @param[in] actions
1470  *   Pointer to the list of actions.
1471  * @param[out] item_flags
1472  *   Pointer to bit mask of all items detected.
1473  * @param[out] action_flags
1474  *   Pointer to bit mask of all actions detected.
1475  * @param[out] error
1476  *   Pointer to the error structure.
1477  *
1478  * @return
1479  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1480  *   is set.
1481  */
1482 static struct mlx5_flow *
1483 flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
1484                    const struct rte_flow_item items[],
1485                    const struct rte_flow_action actions[],
1486                    uint64_t *item_flags,
1487                    uint64_t *action_flags,
1488                    struct rte_flow_error *error)
1489 {
1490         uint32_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
1491         struct mlx5_flow *flow;
1492
1493         size += flow_verbs_get_actions_and_size(actions, action_flags);
1494         size += flow_verbs_get_items_and_size(items, item_flags);
1495         flow = rte_calloc(__func__, 1, size, 0);
1496         if (!flow) {
1497                 rte_flow_error_set(error, ENOMEM,
1498                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1499                                    "not enough memory to create flow");
1500                 return NULL;
1501         }
1502         flow->verbs.attr = (void *)(flow + 1);
1503         flow->verbs.specs =
1504                 (uint8_t *)(flow + 1) + sizeof(struct ibv_flow_attr);
1505         return flow;
1506 }
1507
1508 /**
1509  * Fill the flow with verb spec.
1510  *
1511  * @param[in] dev
1512  *   Pointer to Ethernet device.
1513  * @param[in, out] dev_flow
1514  *   Pointer to the mlx5 flow.
1515  * @param[in] attr
1516  *   Pointer to the flow attributes.
1517  * @param[in] items
1518  *   Pointer to the list of items.
1519  * @param[in] actions
1520  *   Pointer to the list of actions.
1521  * @param[out] error
1522  *   Pointer to the error structure.
1523  *
1524  * @return
1525  *   0 on success, else a negative errno value otherwise and rte_ernno is set.
1526  */
1527 static int
1528 flow_verbs_translate(struct rte_eth_dev *dev,
1529                      struct mlx5_flow *dev_flow,
1530                      const struct rte_flow_attr *attr,
1531                      const struct rte_flow_item items[],
1532                      const struct rte_flow_action actions[],
1533                      struct rte_flow_error *error)
1534 {
1535         uint64_t action_flags = 0;
1536         uint64_t item_flags = 0;
1537         uint64_t priority = attr->priority;
1538         struct priv *priv = dev->data->dev_private;
1539
1540         if (priority == MLX5_FLOW_PRIO_RSVD)
1541                 priority = priv->config.flow_prio - 1;
1542         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1543                 int ret;
1544                 switch (actions->type) {
1545                 case RTE_FLOW_ACTION_TYPE_VOID:
1546                         break;
1547                 case RTE_FLOW_ACTION_TYPE_FLAG:
1548                         flow_verbs_translate_action_flag(actions,
1549                                                          &action_flags,
1550                                                          dev_flow);
1551                         break;
1552                 case RTE_FLOW_ACTION_TYPE_MARK:
1553                         flow_verbs_translate_action_mark(actions,
1554                                                          &action_flags,
1555                                                          dev_flow);
1556                         break;
1557                 case RTE_FLOW_ACTION_TYPE_DROP:
1558                         flow_verbs_translate_action_drop(&action_flags,
1559                                                          dev_flow);
1560                         break;
1561                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1562                         flow_verbs_translate_action_queue(actions,
1563                                                           &action_flags,
1564                                                           dev_flow);
1565                         break;
1566                 case RTE_FLOW_ACTION_TYPE_RSS:
1567                         flow_verbs_translate_action_rss(actions,
1568                                                         &action_flags,
1569                                                         dev_flow);
1570                         break;
1571                 case RTE_FLOW_ACTION_TYPE_COUNT:
1572                         ret = flow_verbs_translate_action_count(dev,
1573                                                                 actions,
1574                                                                 &action_flags,
1575                                                                 dev_flow,
1576                                                                 error);
1577                         if (ret < 0)
1578                                 return ret;
1579                         break;
1580                 default:
1581                         return rte_flow_error_set(error, ENOTSUP,
1582                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1583                                                   actions,
1584                                                   "action not supported");
1585                 }
1586         }
1587         /* Device flow should have action flags by flow_drv_prepare(). */
1588         assert(dev_flow->flow->actions == action_flags);
1589         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1590                 switch (items->type) {
1591                 case RTE_FLOW_ITEM_TYPE_VOID:
1592                         break;
1593                 case RTE_FLOW_ITEM_TYPE_ETH:
1594                         flow_verbs_translate_item_eth(items, &item_flags,
1595                                                       dev_flow);
1596                         break;
1597                 case RTE_FLOW_ITEM_TYPE_VLAN:
1598                         flow_verbs_translate_item_vlan(items, &item_flags,
1599                                                        dev_flow);
1600                         break;
1601                 case RTE_FLOW_ITEM_TYPE_IPV4:
1602                         flow_verbs_translate_item_ipv4(items, &item_flags,
1603                                                        dev_flow);
1604                         break;
1605                 case RTE_FLOW_ITEM_TYPE_IPV6:
1606                         flow_verbs_translate_item_ipv6(items, &item_flags,
1607                                                        dev_flow);
1608                         break;
1609                 case RTE_FLOW_ITEM_TYPE_UDP:
1610                         flow_verbs_translate_item_udp(items, &item_flags,
1611                                                       dev_flow);
1612                         break;
1613                 case RTE_FLOW_ITEM_TYPE_TCP:
1614                         flow_verbs_translate_item_tcp(items, &item_flags,
1615                                                       dev_flow);
1616                         break;
1617                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1618                         flow_verbs_translate_item_vxlan(items, &item_flags,
1619                                                         dev_flow);
1620                         break;
1621                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1622                         flow_verbs_translate_item_vxlan_gpe(items, &item_flags,
1623                                                             dev_flow);
1624                         break;
1625                 case RTE_FLOW_ITEM_TYPE_GRE:
1626                         flow_verbs_translate_item_gre(items, &item_flags,
1627                                                       dev_flow);
1628                         break;
1629                 case RTE_FLOW_ITEM_TYPE_MPLS:
1630                         flow_verbs_translate_item_mpls(items, &item_flags,
1631                                                        dev_flow);
1632                         break;
1633                 default:
1634                         return rte_flow_error_set(error, ENOTSUP,
1635                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1636                                                   NULL,
1637                                                   "item not supported");
1638                 }
1639         }
1640         dev_flow->verbs.attr->priority =
1641                 mlx5_flow_adjust_priority(dev, priority,
1642                                           dev_flow->verbs.attr->priority);
1643         return 0;
1644 }
1645
1646 /**
1647  * Remove the flow from the NIC but keeps it in memory.
1648  *
1649  * @param[in] dev
1650  *   Pointer to the Ethernet device structure.
1651  * @param[in, out] flow
1652  *   Pointer to flow structure.
1653  */
1654 static void
1655 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1656 {
1657         struct mlx5_flow_verbs *verbs;
1658         struct mlx5_flow *dev_flow;
1659
1660         if (!flow)
1661                 return;
1662         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1663                 verbs = &dev_flow->verbs;
1664                 if (verbs->flow) {
1665                         claim_zero(mlx5_glue->destroy_flow(verbs->flow));
1666                         verbs->flow = NULL;
1667                 }
1668                 if (verbs->hrxq) {
1669                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1670                                 mlx5_hrxq_drop_release(dev);
1671                         else
1672                                 mlx5_hrxq_release(dev, verbs->hrxq);
1673                         verbs->hrxq = NULL;
1674                 }
1675         }
1676 }
1677
1678 /**
1679  * Remove the flow from the NIC and the memory.
1680  *
1681  * @param[in] dev
1682  *   Pointer to the Ethernet device structure.
1683  * @param[in, out] flow
1684  *   Pointer to flow structure.
1685  */
1686 static void
1687 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1688 {
1689         struct mlx5_flow *dev_flow;
1690
1691         if (!flow)
1692                 return;
1693         flow_verbs_remove(dev, flow);
1694         while (!LIST_EMPTY(&flow->dev_flows)) {
1695                 dev_flow = LIST_FIRST(&flow->dev_flows);
1696                 LIST_REMOVE(dev_flow, next);
1697                 rte_free(dev_flow);
1698         }
1699         if (flow->counter) {
1700                 flow_verbs_counter_release(flow->counter);
1701                 flow->counter = NULL;
1702         }
1703 }
1704
1705 /**
1706  * Apply the flow to the NIC.
1707  *
1708  * @param[in] dev
1709  *   Pointer to the Ethernet device structure.
1710  * @param[in, out] flow
1711  *   Pointer to flow structure.
1712  * @param[out] error
1713  *   Pointer to error structure.
1714  *
1715  * @return
1716  *   0 on success, a negative errno value otherwise and rte_errno is set.
1717  */
1718 static int
1719 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1720                  struct rte_flow_error *error)
1721 {
1722         struct mlx5_flow_verbs *verbs;
1723         struct mlx5_flow *dev_flow;
1724         int err;
1725
1726         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1727                 verbs = &dev_flow->verbs;
1728                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1729                         verbs->hrxq = mlx5_hrxq_drop_new(dev);
1730                         if (!verbs->hrxq) {
1731                                 rte_flow_error_set
1732                                         (error, errno,
1733                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1734                                          "cannot get drop hash queue");
1735                                 goto error;
1736                         }
1737                 } else {
1738                         struct mlx5_hrxq *hrxq;
1739
1740                         hrxq = mlx5_hrxq_get(dev, flow->key,
1741                                              MLX5_RSS_HASH_KEY_LEN,
1742                                              verbs->hash_fields,
1743                                              (*flow->queue),
1744                                              flow->rss.queue_num);
1745                         if (!hrxq)
1746                                 hrxq = mlx5_hrxq_new(dev, flow->key,
1747                                                      MLX5_RSS_HASH_KEY_LEN,
1748                                                      verbs->hash_fields,
1749                                                      (*flow->queue),
1750                                                      flow->rss.queue_num,
1751                                                      !!(dev_flow->layers &
1752                                                       MLX5_FLOW_LAYER_TUNNEL));
1753                         if (!hrxq) {
1754                                 rte_flow_error_set
1755                                         (error, rte_errno,
1756                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1757                                          "cannot get hash queue");
1758                                 goto error;
1759                         }
1760                         verbs->hrxq = hrxq;
1761                 }
1762                 verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp,
1763                                                      verbs->attr);
1764                 if (!verbs->flow) {
1765                         rte_flow_error_set(error, errno,
1766                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1767                                            NULL,
1768                                            "hardware refuses to create flow");
1769                         goto error;
1770                 }
1771         }
1772         return 0;
1773 error:
1774         err = rte_errno; /* Save rte_errno before cleanup. */
1775         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1776                 verbs = &dev_flow->verbs;
1777                 if (verbs->hrxq) {
1778                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1779                                 mlx5_hrxq_drop_release(dev);
1780                         else
1781                                 mlx5_hrxq_release(dev, verbs->hrxq);
1782                         verbs->hrxq = NULL;
1783                 }
1784         }
1785         rte_errno = err; /* Restore rte_errno. */
1786         return -rte_errno;
1787 }
1788
1789 /**
1790  * Query a flow.
1791  *
1792  * @see rte_flow_query()
1793  * @see rte_flow_ops
1794  */
1795 static int
1796 flow_verbs_query(struct rte_eth_dev *dev,
1797                  struct rte_flow *flow,
1798                  const struct rte_flow_action *actions,
1799                  void *data,
1800                  struct rte_flow_error *error)
1801 {
1802         int ret = -EINVAL;
1803
1804         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1805                 switch (actions->type) {
1806                 case RTE_FLOW_ACTION_TYPE_VOID:
1807                         break;
1808                 case RTE_FLOW_ACTION_TYPE_COUNT:
1809                         ret = flow_verbs_counter_query(dev, flow, data, error);
1810                         break;
1811                 default:
1812                         return rte_flow_error_set(error, ENOTSUP,
1813                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1814                                                   actions,
1815                                                   "action not supported");
1816                 }
1817         }
1818         return ret;
1819 }
1820
1821 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1822         .validate = flow_verbs_validate,
1823         .prepare = flow_verbs_prepare,
1824         .translate = flow_verbs_translate,
1825         .apply = flow_verbs_apply,
1826         .remove = flow_verbs_remove,
1827         .destroy = flow_verbs_destroy,
1828         .query = flow_verbs_query,
1829 };