drivers/net/mlx5/mlx5_flow_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <netinet/in.h>
   6 #include <sys/queue.h>
   7 #include <stdalign.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_eth_ctrl.h>
  24 #include <rte_ethdev_driver.h>
  25 #include <rte_flow.h>
  26 #include <rte_flow_driver.h>
  27 #include <rte_malloc.h>
  28 #include <rte_ip.h>
  29
  30 #include "mlx5.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33 #include "mlx5_glue.h"
  34 #include "mlx5_flow.h"
  35
  36 /**
  37  * Create Verbs flow counter with Verbs library.
  38  *
  39  * @param[in] dev
  40  *   Pointer to the Ethernet device structure.
  41  * @param[in, out] counter
  42  *   mlx5 flow counter object, contains the counter id,
  43  *   handle of created Verbs flow counter is returned
  44  *   in cs field (if counters are supported).
  45  *
  46  * @return
  47  *   0 On success else a negative errno value is returned
  48  *   and rte_errno is set.
  49  */
  50 static int
  51 flow_verbs_counter_create(struct rte_eth_dev *dev,
  52                           struct mlx5_flow_counter *counter)
  53 {
  54 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
  55         struct priv *priv = dev->data->dev_private;
  56         struct ibv_counter_set_init_attr init = {
  57                          .counter_set_id = counter->id};
  58
  59         counter->cs = mlx5_glue->create_counter_set(priv->ctx, &init);
  60         if (!counter->cs) {
  61                 rte_errno = ENOTSUP;
  62                 return -ENOTSUP;
  63         }
  64         return 0;
  65 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
  66         struct priv *priv = dev->data->dev_private;
  67         struct ibv_counters_init_attr init = {0};
  68         struct ibv_counter_attach_attr attach = {0};
  69         int ret;
  70
  71         counter->cs = mlx5_glue->create_counters(priv->ctx, &init);
  72         if (!counter->cs) {
  73                 rte_errno = ENOTSUP;
  74                 return -ENOTSUP;
  75         }
  76         attach.counter_desc = IBV_COUNTER_PACKETS;
  77         attach.index = 0;
  78         ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
  79         if (!ret) {
  80                 attach.counter_desc = IBV_COUNTER_BYTES;
  81                 attach.index = 1;
  82                 ret = mlx5_glue->attach_counters
  83                                         (counter->cs, &attach, NULL);
  84         }
  85         if (ret) {
  86                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
  87                 counter->cs = NULL;
  88                 rte_errno = ret;
  89                 return -ret;
  90         }
  91         return 0;
  92 #else
  93         (void)dev;
  94         (void)counter;
  95         rte_errno = ENOTSUP;
  96         return -ENOTSUP;
  97 #endif
  98 }
  99
 100 /**
 101  * Get a flow counter.
 102  *
 103  * @param[in] dev
 104  *   Pointer to the Ethernet device structure.
 105  * @param[in] shared
 106  *   Indicate if this counter is shared with other flows.
 107  * @param[in] id
 108  *   Counter identifier.
 109  *
 110  * @return
 111  *   A pointer to the counter, NULL otherwise and rte_errno is set.
 112  */
 113 static struct mlx5_flow_counter *
 114 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 115 {
 116         struct priv *priv = dev->data->dev_private;
 117         struct mlx5_flow_counter *cnt;
 118         int ret;
 119
 120         LIST_FOREACH(cnt, &priv->flow_counters, next) {
 121                 if (!cnt->shared || cnt->shared != shared)
 122                         continue;
 123                 if (cnt->id != id)
 124                         continue;
 125                 cnt->ref_cnt++;
 126                 return cnt;
 127         }
 128         cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 129         if (!cnt) {
 130                 rte_errno = ENOMEM;
 131                 return NULL;
 132         }
 133         cnt->id = id;
 134         cnt->shared = shared;
 135         cnt->ref_cnt = 1;
 136         cnt->hits = 0;
 137         cnt->bytes = 0;
 138         /* Create counter with Verbs. */
 139         ret = flow_verbs_counter_create(dev, cnt);
 140         if (!ret) {
 141                 LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
 142                 return cnt;
 143         }
 144         /* Some error occurred in Verbs library. */
 145         rte_free(cnt);
 146         rte_errno = -ret;
 147         return NULL;
 148 }
 149
 150 /**
 151  * Release a flow counter.
 152  *
 153  * @param[in] counter
 154  *   Pointer to the counter handler.
 155  */
 156 static void
 157 flow_verbs_counter_release(struct mlx5_flow_counter *counter)
 158 {
 159         if (--counter->ref_cnt == 0) {
 160 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 161                 claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
 162 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 163                 claim_zero(mlx5_glue->destroy_counters(counter->cs));
 164 #endif
 165                 LIST_REMOVE(counter, next);
 166                 rte_free(counter);
 167         }
 168 }
 169
 170 /**
 171  * Query a flow counter via Verbs library call.
 172  *
 173  * @see rte_flow_query()
 174  * @see rte_flow_ops
 175  */
 176 static int
 177 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
 178                          struct rte_flow *flow, void *data,
 179                          struct rte_flow_error *error)
 180 {
 181 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
 182         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 183         if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 184                 struct rte_flow_query_count *qc = data;
 185                 uint64_t counters[2] = {0, 0};
 186 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 187                 struct ibv_query_counter_set_attr query_cs_attr = {
 188                         .cs = flow->counter->cs,
 189                         .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
 190                 };
 191                 struct ibv_counter_set_data query_out = {
 192                         .out = counters,
 193                         .outlen = 2 * sizeof(uint64_t),
 194                 };
 195                 int err = mlx5_glue->query_counter_set(&query_cs_attr,
 196                                                        &query_out);
 197 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 198                 int err = mlx5_glue->query_counters
 199                                (flow->counter->cs, counters,
 200                                 RTE_DIM(counters),
 201                                 IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
 202 #endif
 203                 if (err)
 204                         return rte_flow_error_set
 205                                 (error, err,
 206                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 207                                  NULL,
 208                                  "cannot read counter");
 209                 qc->hits_set = 1;
 210                 qc->bytes_set = 1;
 211                 qc->hits = counters[0] - flow->counter->hits;
 212                 qc->bytes = counters[1] - flow->counter->bytes;
 213                 if (qc->reset) {
 214                         flow->counter->hits = counters[0];
 215                         flow->counter->bytes = counters[1];
 216                 }
 217                 return 0;
 218         }
 219         return rte_flow_error_set(error, EINVAL,
 220                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 221                                   NULL,
 222                                   "flow does not have counter");
 223 #else
 224         (void)flow;
 225         (void)data;
 226         return rte_flow_error_set(error, ENOTSUP,
 227                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 228                                   NULL,
 229                                   "counters are not available");
 230 #endif
 231 }
 232
 233 /**
 234  * Add a verbs item specification into @p flow.
 235  *
 236  * @param[in, out] flow
 237  *   Pointer to flow structure.
 238  * @param[in] src
 239  *   Create specification.
 240  * @param[in] size
 241  *   Size in bytes of the specification to copy.
 242  */
 243 static void
 244 flow_verbs_spec_add(struct mlx5_flow *flow, void *src, unsigned int size)
 245 {
 246         struct mlx5_flow_verbs *verbs = &flow->verbs;
 247
 248         if (verbs->specs) {
 249                 void *dst;
 250
 251                 dst = (void *)(verbs->specs + verbs->size);
 252                 memcpy(dst, src, size);
 253                 ++verbs->attr->num_of_specs;
 254         }
 255         verbs->size += size;
 256 }
 257
 258 /**
 259  * Convert the @p item into a Verbs specification. This function assumes that
 260  * the input is valid and that there is space to insert the requested item
 261  * into the flow.
 262  *
 263  * @param[in] item
 264  *   Item specification.
 265  * @param[in] item_flags
 266  *   Bit field with all detected items.
 267  * @param[in, out] dev_flow
 268  *   Pointer to dev_flow structure.
 269  */
 270 static void
 271 flow_verbs_translate_item_eth(const struct rte_flow_item *item,
 272                               uint64_t *item_flags,
 273                               struct mlx5_flow *dev_flow)
 274 {
 275         const struct rte_flow_item_eth *spec = item->spec;
 276         const struct rte_flow_item_eth *mask = item->mask;
 277         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 278         const unsigned int size = sizeof(struct ibv_flow_spec_eth);
 279         struct ibv_flow_spec_eth eth = {
 280                 .type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 281                 .size = size,
 282         };
 283
 284         if (!mask)
 285                 mask = &rte_flow_item_eth_mask;
 286         if (spec) {
 287                 unsigned int i;
 288
 289                 memcpy(&eth.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
 290                 memcpy(&eth.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
 291                 eth.val.ether_type = spec->type;
 292                 memcpy(&eth.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
 293                 memcpy(&eth.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
 294                 eth.mask.ether_type = mask->type;
 295                 /* Remove unwanted bits from values. */
 296                 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
 297                         eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
 298                         eth.val.src_mac[i] &= eth.mask.src_mac[i];
 299                 }
 300                 eth.val.ether_type &= eth.mask.ether_type;
 301                 dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 302         }
 303         flow_verbs_spec_add(dev_flow, &eth, size);
 304         *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 305                                 MLX5_FLOW_LAYER_OUTER_L2;
 306 }
 307
 308 /**
 309  * Update the VLAN tag in the Verbs Ethernet specification.
 310  * This function assumes that the input is valid and there is space to add
 311  * the requested item.
 312  *
 313  * @param[in, out] attr
 314  *   Pointer to Verbs attributes structure.
 315  * @param[in] eth
 316  *   Verbs structure containing the VLAN information to copy.
 317  */
 318 static void
 319 flow_verbs_item_vlan_update(struct ibv_flow_attr *attr,
 320                             struct ibv_flow_spec_eth *eth)
 321 {
 322         unsigned int i;
 323         const enum ibv_flow_spec_type search = eth->type;
 324         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 325                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 326
 327         for (i = 0; i != attr->num_of_specs; ++i) {
 328                 if (hdr->type == search) {
 329                         struct ibv_flow_spec_eth *e =
 330                                 (struct ibv_flow_spec_eth *)hdr;
 331
 332                         e->val.vlan_tag = eth->val.vlan_tag;
 333                         e->mask.vlan_tag = eth->mask.vlan_tag;
 334                         e->val.ether_type = eth->val.ether_type;
 335                         e->mask.ether_type = eth->mask.ether_type;
 336                         break;
 337                 }
 338                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 339         }
 340 }
 341
 342 /**
 343  * Convert the @p item into a Verbs specification. This function assumes that
 344  * the input is valid and that there is space to insert the requested item
 345  * into the flow.
 346  *
 347  * @param[in] item
 348  *   Item specification.
 349  * @param[in, out] item_flags
 350  *   Bit mask that holds all detected items.
 351  * @param[in, out] dev_flow
 352  *   Pointer to dev_flow structure.
 353  */
 354 static void
 355 flow_verbs_translate_item_vlan(const struct rte_flow_item *item,
 356                                uint64_t *item_flags,
 357                                struct mlx5_flow *dev_flow)
 358 {
 359         const struct rte_flow_item_vlan *spec = item->spec;
 360         const struct rte_flow_item_vlan *mask = item->mask;
 361         unsigned int size = sizeof(struct ibv_flow_spec_eth);
 362         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 363         struct ibv_flow_spec_eth eth = {
 364                 .type = IBV_FLOW_SPEC_ETH | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 365                 .size = size,
 366         };
 367         const uint32_t l2m = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
 368                                       MLX5_FLOW_LAYER_OUTER_L2;
 369
 370         if (!mask)
 371                 mask = &rte_flow_item_vlan_mask;
 372         if (spec) {
 373                 eth.val.vlan_tag = spec->tci;
 374                 eth.mask.vlan_tag = mask->tci;
 375                 eth.val.vlan_tag &= eth.mask.vlan_tag;
 376                 eth.val.ether_type = spec->inner_type;
 377                 eth.mask.ether_type = mask->inner_type;
 378                 eth.val.ether_type &= eth.mask.ether_type;
 379         }
 380         if (!(*item_flags & l2m)) {
 381                 dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 382                 flow_verbs_spec_add(dev_flow, &eth, size);
 383         } else {
 384                 flow_verbs_item_vlan_update(dev_flow->verbs.attr, &eth);
 385                 size = 0; /* Only an update is done in eth specification. */
 386         }
 387         *item_flags |= tunnel ?
 388                        (MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_VLAN) :
 389                        (MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_OUTER_VLAN);
 390 }
 391
 392 /**
 393  * Convert the @p item into a Verbs specification. This function assumes that
 394  * the input is valid and that there is space to insert the requested item
 395  * into the flow.
 396  *
 397  * @param[in] item
 398  *   Item specification.
 399  * @param[in, out] item_flags
 400  *   Bit mask that marks all detected items.
 401  * @param[in, out] dev_flow
 402  *   Pointer to sepacific flow structure.
 403  */
 404 static void
 405 flow_verbs_translate_item_ipv4(const struct rte_flow_item *item,
 406                                uint64_t *item_flags,
 407                                struct mlx5_flow *dev_flow)
 408 {
 409         const struct rte_flow_item_ipv4 *spec = item->spec;
 410         const struct rte_flow_item_ipv4 *mask = item->mask;
 411         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 412         unsigned int size = sizeof(struct ibv_flow_spec_ipv4_ext);
 413         struct ibv_flow_spec_ipv4_ext ipv4 = {
 414                 .type = IBV_FLOW_SPEC_IPV4_EXT |
 415                         (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 416                 .size = size,
 417         };
 418
 419         if (!mask)
 420                 mask = &rte_flow_item_ipv4_mask;
 421         *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
 422                                 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
 423         if (spec) {
 424                 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
 425                         .src_ip = spec->hdr.src_addr,
 426                         .dst_ip = spec->hdr.dst_addr,
 427                         .proto = spec->hdr.next_proto_id,
 428                         .tos = spec->hdr.type_of_service,
 429                 };
 430                 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
 431                         .src_ip = mask->hdr.src_addr,
 432                         .dst_ip = mask->hdr.dst_addr,
 433                         .proto = mask->hdr.next_proto_id,
 434                         .tos = mask->hdr.type_of_service,
 435                 };
 436                 /* Remove unwanted bits from values. */
 437                 ipv4.val.src_ip &= ipv4.mask.src_ip;
 438                 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
 439                 ipv4.val.proto &= ipv4.mask.proto;
 440                 ipv4.val.tos &= ipv4.mask.tos;
 441         }
 442         dev_flow->verbs.hash_fields |=
 443                 mlx5_flow_hashfields_adjust(dev_flow, tunnel,
 444                                             MLX5_IPV4_LAYER_TYPES,
 445                                             MLX5_IPV4_IBV_RX_HASH);
 446         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L3;
 447         flow_verbs_spec_add(dev_flow, &ipv4, size);
 448 }
 449
 450 /**
 451  * Convert the @p item into a Verbs specification. This function assumes that
 452  * the input is valid and that there is space to insert the requested item
 453  * into the flow.
 454  *
 455  * @param[in] item
 456  *   Item specification.
 457  * @param[in, out] item_flags
 458  *   Bit mask that marks all detected items.
 459  * @param[in, out] dev_flow
 460  *   Pointer to sepacific flow structure.
 461  */
 462 static void
 463 flow_verbs_translate_item_ipv6(const struct rte_flow_item *item,
 464                                uint64_t *item_flags,
 465                                struct mlx5_flow *dev_flow)
 466 {
 467         const struct rte_flow_item_ipv6 *spec = item->spec;
 468         const struct rte_flow_item_ipv6 *mask = item->mask;
 469         const int tunnel = !!(dev_flow->flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 470         unsigned int size = sizeof(struct ibv_flow_spec_ipv6);
 471         struct ibv_flow_spec_ipv6 ipv6 = {
 472                 .type = IBV_FLOW_SPEC_IPV6 | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 473                 .size = size,
 474         };
 475
 476         if (!mask)
 477                 mask = &rte_flow_item_ipv6_mask;
 478          *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
 479                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
 480         if (spec) {
 481                 unsigned int i;
 482                 uint32_t vtc_flow_val;
 483                 uint32_t vtc_flow_mask;
 484
 485                 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
 486                        RTE_DIM(ipv6.val.src_ip));
 487                 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
 488                        RTE_DIM(ipv6.val.dst_ip));
 489                 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
 490                        RTE_DIM(ipv6.mask.src_ip));
 491                 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
 492                        RTE_DIM(ipv6.mask.dst_ip));
 493                 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
 494                 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
 495                 ipv6.val.flow_label =
 496                         rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
 497                                          IPV6_HDR_FL_SHIFT);
 498                 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
 499                                          IPV6_HDR_TC_SHIFT;
 500                 ipv6.val.next_hdr = spec->hdr.proto;
 501                 ipv6.val.hop_limit = spec->hdr.hop_limits;
 502                 ipv6.mask.flow_label =
 503                         rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
 504                                          IPV6_HDR_FL_SHIFT);
 505                 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
 506                                           IPV6_HDR_TC_SHIFT;
 507                 ipv6.mask.next_hdr = mask->hdr.proto;
 508                 ipv6.mask.hop_limit = mask->hdr.hop_limits;
 509                 /* Remove unwanted bits from values. */
 510                 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
 511                         ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
 512                         ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
 513                 }
 514                 ipv6.val.flow_label &= ipv6.mask.flow_label;
 515                 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
 516                 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
 517                 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
 518         }
 519         dev_flow->verbs.hash_fields |=
 520                 mlx5_flow_hashfields_adjust(dev_flow, tunnel,
 521                                             MLX5_IPV6_LAYER_TYPES,
 522                                             MLX5_IPV6_IBV_RX_HASH);
 523         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L3;
 524         flow_verbs_spec_add(dev_flow, &ipv6, size);
 525 }
 526
 527 /**
 528  * Convert the @p item into a Verbs specification. This function assumes that
 529  * the input is valid and that there is space to insert the requested item
 530  * into the flow.
 531  *
 532  * @param[in] item
 533  *   Item specification.
 534  * @param[in, out] item_flags
 535  *   Bit mask that marks all detected items.
 536  * @param[in, out] dev_flow
 537  *   Pointer to sepacific flow structure.
 538  */
 539 static void
 540 flow_verbs_translate_item_udp(const struct rte_flow_item *item,
 541                               uint64_t *item_flags,
 542                               struct mlx5_flow *dev_flow)
 543 {
 544         const struct rte_flow_item_udp *spec = item->spec;
 545         const struct rte_flow_item_udp *mask = item->mask;
 546         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
 547         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 548         struct ibv_flow_spec_tcp_udp udp = {
 549                 .type = IBV_FLOW_SPEC_UDP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 550                 .size = size,
 551         };
 552
 553         if (!mask)
 554                 mask = &rte_flow_item_udp_mask;
 555         *item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
 556                                 MLX5_FLOW_LAYER_OUTER_L4_UDP;
 557         if (spec) {
 558                 udp.val.dst_port = spec->hdr.dst_port;
 559                 udp.val.src_port = spec->hdr.src_port;
 560                 udp.mask.dst_port = mask->hdr.dst_port;
 561                 udp.mask.src_port = mask->hdr.src_port;
 562                 /* Remove unwanted bits from values. */
 563                 udp.val.src_port &= udp.mask.src_port;
 564                 udp.val.dst_port &= udp.mask.dst_port;
 565         }
 566         dev_flow->verbs.hash_fields |=
 567                 mlx5_flow_hashfields_adjust(dev_flow, tunnel, ETH_RSS_UDP,
 568                                             (IBV_RX_HASH_SRC_PORT_UDP |
 569                                              IBV_RX_HASH_DST_PORT_UDP));
 570         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L4;
 571         flow_verbs_spec_add(dev_flow, &udp, size);
 572 }
 573
 574 /**
 575  * Convert the @p item into a Verbs specification. This function assumes that
 576  * the input is valid and that there is space to insert the requested item
 577  * into the flow.
 578  *
 579  * @param[in] item
 580  *   Item specification.
 581  * @param[in, out] item_flags
 582  *   Bit mask that marks all detected items.
 583  * @param[in, out] dev_flow
 584  *   Pointer to sepacific flow structure.
 585  */
 586 static void
 587 flow_verbs_translate_item_tcp(const struct rte_flow_item *item,
 588                               uint64_t *item_flags,
 589                               struct mlx5_flow *dev_flow)
 590 {
 591         const struct rte_flow_item_tcp *spec = item->spec;
 592         const struct rte_flow_item_tcp *mask = item->mask;
 593         const int tunnel = !!(dev_flow->flow->layers & MLX5_FLOW_LAYER_TUNNEL);
 594         unsigned int size = sizeof(struct ibv_flow_spec_tcp_udp);
 595         struct ibv_flow_spec_tcp_udp tcp = {
 596                 .type = IBV_FLOW_SPEC_TCP | (tunnel ? IBV_FLOW_SPEC_INNER : 0),
 597                 .size = size,
 598         };
 599
 600         if (!mask)
 601                 mask = &rte_flow_item_tcp_mask;
 602         *item_flags |=  tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
 603                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
 604         if (spec) {
 605                 tcp.val.dst_port = spec->hdr.dst_port;
 606                 tcp.val.src_port = spec->hdr.src_port;
 607                 tcp.mask.dst_port = mask->hdr.dst_port;
 608                 tcp.mask.src_port = mask->hdr.src_port;
 609                 /* Remove unwanted bits from values. */
 610                 tcp.val.src_port &= tcp.mask.src_port;
 611                 tcp.val.dst_port &= tcp.mask.dst_port;
 612         }
 613         dev_flow->verbs.hash_fields |=
 614                 mlx5_flow_hashfields_adjust(dev_flow, tunnel, ETH_RSS_TCP,
 615                                             (IBV_RX_HASH_SRC_PORT_TCP |
 616                                              IBV_RX_HASH_DST_PORT_TCP));
 617         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L4;
 618         flow_verbs_spec_add(dev_flow, &tcp, size);
 619 }
 620
 621 /**
 622  * Convert the @p item into a Verbs specification. This function assumes that
 623  * the input is valid and that there is space to insert the requested item
 624  * into the flow.
 625  *
 626  * @param[in] item
 627  *   Item specification.
 628  * @param[in, out] item_flags
 629  *   Bit mask that marks all detected items.
 630  * @param[in, out] dev_flow
 631  *   Pointer to sepacific flow structure.
 632  */
 633 static void
 634 flow_verbs_translate_item_vxlan(const struct rte_flow_item *item,
 635                                 uint64_t *item_flags,
 636                                 struct mlx5_flow *dev_flow)
 637 {
 638         const struct rte_flow_item_vxlan *spec = item->spec;
 639         const struct rte_flow_item_vxlan *mask = item->mask;
 640         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 641         struct ibv_flow_spec_tunnel vxlan = {
 642                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 643                 .size = size,
 644         };
 645         union vni {
 646                 uint32_t vlan_id;
 647                 uint8_t vni[4];
 648         } id = { .vlan_id = 0, };
 649
 650         if (!mask)
 651                 mask = &rte_flow_item_vxlan_mask;
 652         if (spec) {
 653                 memcpy(&id.vni[1], spec->vni, 3);
 654                 vxlan.val.tunnel_id = id.vlan_id;
 655                 memcpy(&id.vni[1], mask->vni, 3);
 656                 vxlan.mask.tunnel_id = id.vlan_id;
 657                 /* Remove unwanted bits from values. */
 658                 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
 659         }
 660         flow_verbs_spec_add(dev_flow, &vxlan, size);
 661         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 662         *item_flags |= MLX5_FLOW_LAYER_VXLAN;
 663 }
 664
 665 /**
 666  * Convert the @p item into a Verbs specification. This function assumes that
 667  * the input is valid and that there is space to insert the requested item
 668  * into the flow.
 669  *
 670  * @param[in] item
 671  *   Item specification.
 672  * @param[in, out] item_flags
 673  *   Bit mask that marks all detected items.
 674  * @param[in, out] dev_flow
 675  *   Pointer to sepacific flow structure.
 676  */
 677 static void
 678 flow_verbs_translate_item_vxlan_gpe(const struct rte_flow_item *item,
 679                                     uint64_t *item_flags,
 680                                     struct mlx5_flow *dev_flow)
 681 {
 682         const struct rte_flow_item_vxlan_gpe *spec = item->spec;
 683         const struct rte_flow_item_vxlan_gpe *mask = item->mask;
 684         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 685         struct ibv_flow_spec_tunnel vxlan_gpe = {
 686                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 687                 .size = size,
 688         };
 689         union vni {
 690                 uint32_t vlan_id;
 691                 uint8_t vni[4];
 692         } id = { .vlan_id = 0, };
 693
 694         if (!mask)
 695                 mask = &rte_flow_item_vxlan_gpe_mask;
 696         if (spec) {
 697                 memcpy(&id.vni[1], spec->vni, 3);
 698                 vxlan_gpe.val.tunnel_id = id.vlan_id;
 699                 memcpy(&id.vni[1], mask->vni, 3);
 700                 vxlan_gpe.mask.tunnel_id = id.vlan_id;
 701                 /* Remove unwanted bits from values. */
 702                 vxlan_gpe.val.tunnel_id &= vxlan_gpe.mask.tunnel_id;
 703         }
 704         flow_verbs_spec_add(dev_flow, &vxlan_gpe, size);
 705         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 706         *item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
 707 }
 708
 709 /**
 710  * Update the protocol in Verbs IPv4/IPv6 spec.
 711  *
 712  * @param[in, out] attr
 713  *   Pointer to Verbs attributes structure.
 714  * @param[in] search
 715  *   Specification type to search in order to update the IP protocol.
 716  * @param[in] protocol
 717  *   Protocol value to set if none is present in the specification.
 718  */
 719 static void
 720 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr *attr,
 721                                        enum ibv_flow_spec_type search,
 722                                        uint8_t protocol)
 723 {
 724         unsigned int i;
 725         struct ibv_spec_header *hdr = (struct ibv_spec_header *)
 726                 ((uint8_t *)attr + sizeof(struct ibv_flow_attr));
 727
 728         if (!attr)
 729                 return;
 730         for (i = 0; i != attr->num_of_specs; ++i) {
 731                 if (hdr->type == search) {
 732                         union {
 733                                 struct ibv_flow_spec_ipv4_ext *ipv4;
 734                                 struct ibv_flow_spec_ipv6 *ipv6;
 735                         } ip;
 736
 737                         switch (search) {
 738                         case IBV_FLOW_SPEC_IPV4_EXT:
 739                                 ip.ipv4 = (struct ibv_flow_spec_ipv4_ext *)hdr;
 740                                 if (!ip.ipv4->val.proto) {
 741                                         ip.ipv4->val.proto = protocol;
 742                                         ip.ipv4->mask.proto = 0xff;
 743                                 }
 744                                 break;
 745                         case IBV_FLOW_SPEC_IPV6:
 746                                 ip.ipv6 = (struct ibv_flow_spec_ipv6 *)hdr;
 747                                 if (!ip.ipv6->val.next_hdr) {
 748                                         ip.ipv6->val.next_hdr = protocol;
 749                                         ip.ipv6->mask.next_hdr = 0xff;
 750                                 }
 751                                 break;
 752                         default:
 753                                 break;
 754                         }
 755                         break;
 756                 }
 757                 hdr = (struct ibv_spec_header *)((uint8_t *)hdr + hdr->size);
 758         }
 759 }
 760
 761 /**
 762  * Convert the @p item into a Verbs specification. This function assumes that
 763  * the input is valid and that there is space to insert the requested item
 764  * into the flow.
 765  *
 766  * @param[in] item
 767  *   Item specification.
 768  * @param[in, out] item_flags
 769  *   Bit mask that marks all detected items.
 770  * @param[in, out] dev_flow
 771  *   Pointer to sepacific flow structure.
 772  */
 773 static void
 774 flow_verbs_translate_item_gre(const struct rte_flow_item *item __rte_unused,
 775                               uint64_t *item_flags,
 776                               struct mlx5_flow *dev_flow)
 777 {
 778         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
 779 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
 780         unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
 781         struct ibv_flow_spec_tunnel tunnel = {
 782                 .type = IBV_FLOW_SPEC_VXLAN_TUNNEL,
 783                 .size = size,
 784         };
 785 #else
 786         const struct rte_flow_item_gre *spec = item->spec;
 787         const struct rte_flow_item_gre *mask = item->mask;
 788         unsigned int size = sizeof(struct ibv_flow_spec_gre);
 789         struct ibv_flow_spec_gre tunnel = {
 790                 .type = IBV_FLOW_SPEC_GRE,
 791                 .size = size,
 792         };
 793
 794         if (!mask)
 795                 mask = &rte_flow_item_gre_mask;
 796         if (spec) {
 797                 tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
 798                 tunnel.val.protocol = spec->protocol;
 799                 tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
 800                 tunnel.mask.protocol = mask->protocol;
 801                 /* Remove unwanted bits from values. */
 802                 tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
 803                 tunnel.val.protocol &= tunnel.mask.protocol;
 804                 tunnel.val.key &= tunnel.mask.key;
 805         }
 806 #endif
 807         if (*item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 808                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 809                                                        IBV_FLOW_SPEC_IPV4_EXT,
 810                                                        IPPROTO_GRE);
 811         else
 812                 flow_verbs_item_gre_ip_protocol_update(verbs->attr,
 813                                                        IBV_FLOW_SPEC_IPV6,
 814                                                        IPPROTO_GRE);
 815         flow_verbs_spec_add(dev_flow, &tunnel, size);
 816         verbs->attr->priority = MLX5_PRIORITY_MAP_L2;
 817         *item_flags |= MLX5_FLOW_LAYER_GRE;
 818 }
 819
 820 /**
 821  * Convert the @p action into a Verbs specification. This function assumes that
 822  * the input is valid and that there is space to insert the requested action
 823  * into the flow. This function also return the action that was added.
 824  *
 825  * @param[in] item
 826  *   Item specification.
 827  * @param[in, out] item_flags
 828  *   Bit mask that marks all detected items.
 829  * @param[in, out] dev_flow
 830  *   Pointer to sepacific flow structure.
 831  */
 832 static void
 833 flow_verbs_translate_item_mpls(const struct rte_flow_item *item __rte_unused,
 834                                uint64_t *action_flags __rte_unused,
 835                                struct mlx5_flow *dev_flow __rte_unused)
 836 {
 837 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
 838         const struct rte_flow_item_mpls *spec = item->spec;
 839         const struct rte_flow_item_mpls *mask = item->mask;
 840         unsigned int size = sizeof(struct ibv_flow_spec_mpls);
 841         struct ibv_flow_spec_mpls mpls = {
 842                 .type = IBV_FLOW_SPEC_MPLS,
 843                 .size = size,
 844         };
 845
 846         if (!mask)
 847                 mask = &rte_flow_item_mpls_mask;
 848         if (spec) {
 849                 memcpy(&mpls.val.label, spec, sizeof(mpls.val.label));
 850                 memcpy(&mpls.mask.label, mask, sizeof(mpls.mask.label));
 851                 /* Remove unwanted bits from values.  */
 852                 mpls.val.label &= mpls.mask.label;
 853         }
 854         flow_verbs_spec_add(dev_flow, &mpls, size);
 855         dev_flow->verbs.attr->priority = MLX5_PRIORITY_MAP_L2;
 856         *action_flags |= MLX5_FLOW_LAYER_MPLS;
 857 #endif
 858 }
 859
 860 /**
 861  * Convert the @p action into a Verbs specification. This function assumes that
 862  * the input is valid and that there is space to insert the requested action
 863  * into the flow. This function also return the action that was added.
 864  *
 865  * @param[in, out] action_flags
 866  *   Pointer to the detected actions.
 867  * @param[in] dev_flow
 868  *   Pointer to mlx5_flow.
 869  */
 870 static void
 871 flow_verbs_translate_action_drop(uint64_t *action_flags,
 872                                  struct mlx5_flow *dev_flow)
 873 {
 874         unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 875         struct ibv_flow_spec_action_drop drop = {
 876                         .type = IBV_FLOW_SPEC_ACTION_DROP,
 877                         .size = size,
 878         };
 879
 880         flow_verbs_spec_add(dev_flow, &drop, size);
 881         *action_flags |= MLX5_FLOW_ACTION_DROP;
 882 }
 883
 884 /**
 885  * Convert the @p action into a Verbs specification. This function assumes that
 886  * the input is valid and that there is space to insert the requested action
 887  * into the flow. This function also return the action that was added.
 888  *
 889  * @param[in] action
 890  *   Action configuration.
 891  * @param[in, out] action_flags
 892  *   Pointer to the detected actions.
 893  * @param[in] dev_flow
 894  *   Pointer to mlx5_flow.
 895  */
 896 static void
 897 flow_verbs_translate_action_queue(const struct rte_flow_action *action,
 898                                   uint64_t *action_flags,
 899                                   struct mlx5_flow *dev_flow)
 900 {
 901         const struct rte_flow_action_queue *queue = action->conf;
 902         struct rte_flow *flow = dev_flow->flow;
 903
 904         if (flow->queue)
 905                 (*flow->queue)[0] = queue->index;
 906         flow->rss.queue_num = 1;
 907         *action_flags |= MLX5_FLOW_ACTION_QUEUE;
 908 }
 909
 910 /**
 911  * Convert the @p action into a Verbs specification. This function assumes that
 912  * the input is valid and that there is space to insert the requested action
 913  * into the flow. This function also return the action that was added.
 914  *
 915  * @param[in] action
 916  *   Action configuration.
 917  * @param[in, out] action_flags
 918  *   Pointer to the detected actions.
 919  * @param[in] dev_flow
 920  *   Pointer to mlx5_flow.
 921  */
 922 static void
 923 flow_verbs_translate_action_rss(const struct rte_flow_action *action,
 924                                 uint64_t *action_flags,
 925                                 struct mlx5_flow *dev_flow)
 926 {
 927         const struct rte_flow_action_rss *rss = action->conf;
 928         struct rte_flow *flow = dev_flow->flow;
 929
 930         if (flow->queue)
 931                 memcpy((*flow->queue), rss->queue,
 932                        rss->queue_num * sizeof(uint16_t));
 933         flow->rss.queue_num = rss->queue_num;
 934         memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
 935         flow->rss.types = rss->types;
 936         flow->rss.level = rss->level;
 937         *action_flags |= MLX5_FLOW_ACTION_RSS;
 938 }
 939
 940 /**
 941  * Convert the @p action into a Verbs specification. This function assumes that
 942  * the input is valid and that there is space to insert the requested action
 943  * into the flow. This function also return the action that was added.
 944  *
 945  * @param[in] action
 946  *   Action configuration.
 947  * @param[in, out] action_flags
 948  *   Pointer to the detected actions.
 949  * @param[in] dev_flow
 950  *   Pointer to mlx5_flow.
 951  */
 952 static void
 953 flow_verbs_translate_action_flag
 954                         (const struct rte_flow_action *action __rte_unused,
 955                          uint64_t *action_flags,
 956                          struct mlx5_flow *dev_flow)
 957 {
 958         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 959         struct ibv_flow_spec_action_tag tag = {
 960                 .type = IBV_FLOW_SPEC_ACTION_TAG,
 961                 .size = size,
 962                 .tag_id = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT),
 963         };
 964         *action_flags |= MLX5_FLOW_ACTION_MARK;
 965         flow_verbs_spec_add(dev_flow, &tag, size);
 966 }
 967
 968 /**
 969  * Update verbs specification to modify the flag to mark.
 970  *
 971  * @param[in, out] verbs
 972  *   Pointer to the mlx5_flow_verbs structure.
 973  * @param[in] mark_id
 974  *   Mark identifier to replace the flag.
 975  */
 976 static void
 977 flow_verbs_mark_update(struct mlx5_flow_verbs *verbs, uint32_t mark_id)
 978 {
 979         struct ibv_spec_header *hdr;
 980         int i;
 981
 982         if (!verbs)
 983                 return;
 984         /* Update Verbs specification. */
 985         hdr = (struct ibv_spec_header *)verbs->specs;
 986         if (!hdr)
 987                 return;
 988         for (i = 0; i != verbs->attr->num_of_specs; ++i) {
 989                 if (hdr->type == IBV_FLOW_SPEC_ACTION_TAG) {
 990                         struct ibv_flow_spec_action_tag *t =
 991                                 (struct ibv_flow_spec_action_tag *)hdr;
 992
 993                         t->tag_id = mlx5_flow_mark_set(mark_id);
 994                 }
 995                 hdr = (struct ibv_spec_header *)((uintptr_t)hdr + hdr->size);
 996         }
 997 }
 998
 999 /**
1000  * Convert the @p action into a Verbs specification. This function assumes that
1001  * the input is valid and that there is space to insert the requested action
1002  * into the flow. This function also return the action that was added.
1003  *
1004  * @param[in] action
1005  *   Action configuration.
1006  * @param[in, out] action_flags
1007  *   Pointer to the detected actions.
1008  * @param[in] dev_flow
1009  *   Pointer to mlx5_flow.
1010  */
1011 static void
1012 flow_verbs_translate_action_mark(const struct rte_flow_action *action,
1013                                  uint64_t *action_flags,
1014                                  struct mlx5_flow *dev_flow)
1015 {
1016         const struct rte_flow_action_mark *mark = action->conf;
1017         unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1018         struct ibv_flow_spec_action_tag tag = {
1019                 .type = IBV_FLOW_SPEC_ACTION_TAG,
1020                 .size = size,
1021         };
1022         struct mlx5_flow_verbs *verbs = &dev_flow->verbs;
1023
1024         if (*action_flags & MLX5_FLOW_ACTION_FLAG) {
1025                 flow_verbs_mark_update(verbs, mark->id);
1026                 size = 0;
1027         } else {
1028                 tag.tag_id = mlx5_flow_mark_set(mark->id);
1029                 flow_verbs_spec_add(dev_flow, &tag, size);
1030         }
1031         *action_flags |= MLX5_FLOW_ACTION_MARK;
1032 }
1033
1034 /**
1035  * Convert the @p action into a Verbs specification. This function assumes that
1036  * the input is valid and that there is space to insert the requested action
1037  * into the flow. This function also return the action that was added.
1038  *
1039  * @param[in] dev
1040  *   Pointer to the Ethernet device structure.
1041  * @param[in] action
1042  *   Action configuration.
1043  * @param[in, out] action_flags
1044  *   Pointer to the detected actions.
1045  * @param[in] dev_flow
1046  *   Pointer to mlx5_flow.
1047  * @param[out] error
1048  *   Pointer to error structure.
1049  *
1050  * @return
1051  *   0 On success else a negative errno value is returned and rte_errno is set.
1052  */
1053 static int
1054 flow_verbs_translate_action_count(struct rte_eth_dev *dev,
1055                                   const struct rte_flow_action *action,
1056                                   uint64_t *action_flags,
1057                                   struct mlx5_flow *dev_flow,
1058                                   struct rte_flow_error *error)
1059 {
1060         const struct rte_flow_action_count *count = action->conf;
1061         struct rte_flow *flow = dev_flow->flow;
1062 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1063         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1064         unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1065         struct ibv_flow_spec_counter_action counter = {
1066                 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1067                 .size = size,
1068         };
1069 #endif
1070
1071         if (!flow->counter) {
1072                 flow->counter = flow_verbs_counter_new(dev, count->shared,
1073                                                        count->id);
1074                 if (!flow->counter)
1075                         return rte_flow_error_set(error, rte_errno,
1076                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1077                                                   action,
1078                                                   "cannot get counter"
1079                                                   " context.");
1080         }
1081         *action_flags |= MLX5_FLOW_ACTION_COUNT;
1082 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
1083         counter.counter_set_handle = flow->counter->cs->handle;
1084         flow_verbs_spec_add(dev_flow, &counter, size);
1085 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1086         counter.counters = flow->counter->cs;
1087         flow_verbs_spec_add(dev_flow, &counter, size);
1088 #endif
1089         return 0;
1090 }
1091
1092 /**
1093  * Internal validation function. For validating both actions and items.
1094  *
1095  * @param[in] dev
1096  *   Pointer to the Ethernet device structure.
1097  * @param[in] attr
1098  *   Pointer to the flow attributes.
1099  * @param[in] items
1100  *   Pointer to the list of items.
1101  * @param[in] actions
1102  *   Pointer to the list of actions.
1103  * @param[out] error
1104  *   Pointer to the error structure.
1105  *
1106  * @return
1107  *   0 on success, a negative errno value otherwise and rte_errno is set.
1108  */
1109 static int
1110 flow_verbs_validate(struct rte_eth_dev *dev,
1111                     const struct rte_flow_attr *attr,
1112                     const struct rte_flow_item items[],
1113                     const struct rte_flow_action actions[],
1114                     struct rte_flow_error *error)
1115 {
1116         int ret;
1117         uint32_t action_flags = 0;
1118         uint32_t item_flags = 0;
1119         int tunnel = 0;
1120         uint8_t next_protocol = 0xff;
1121
1122         if (items == NULL)
1123                 return -1;
1124         ret = mlx5_flow_validate_attributes(dev, attr, error);
1125         if (ret < 0)
1126                 return ret;
1127         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1128                 int ret = 0;
1129                 switch (items->type) {
1130                 case RTE_FLOW_ITEM_TYPE_VOID:
1131                         break;
1132                 case RTE_FLOW_ITEM_TYPE_ETH:
1133                         ret = mlx5_flow_validate_item_eth(items, item_flags,
1134                                                           error);
1135                         if (ret < 0)
1136                                 return ret;
1137                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1138                                                MLX5_FLOW_LAYER_OUTER_L2;
1139                         break;
1140                 case RTE_FLOW_ITEM_TYPE_VLAN:
1141                         ret = mlx5_flow_validate_item_vlan(items, item_flags,
1142                                                            error);
1143                         if (ret < 0)
1144                                 return ret;
1145                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1146                                                MLX5_FLOW_LAYER_OUTER_VLAN;
1147                         break;
1148                 case RTE_FLOW_ITEM_TYPE_IPV4:
1149                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
1150                                                            error);
1151                         if (ret < 0)
1152                                 return ret;
1153                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1154                                                MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1155                         if (items->mask != NULL &&
1156                             ((const struct rte_flow_item_ipv4 *)
1157                              items->mask)->hdr.next_proto_id)
1158                                 next_protocol =
1159                                         ((const struct rte_flow_item_ipv4 *)
1160                                          (items->spec))->hdr.next_proto_id;
1161                         break;
1162                 case RTE_FLOW_ITEM_TYPE_IPV6:
1163                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
1164                                                            error);
1165                         if (ret < 0)
1166                                 return ret;
1167                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1168                                                MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1169                         if (items->mask != NULL &&
1170                             ((const struct rte_flow_item_ipv6 *)
1171                              items->mask)->hdr.proto)
1172                                 next_protocol =
1173                                         ((const struct rte_flow_item_ipv6 *)
1174                                          items->spec)->hdr.proto;
1175                         break;
1176                 case RTE_FLOW_ITEM_TYPE_UDP:
1177                         ret = mlx5_flow_validate_item_udp(items, item_flags,
1178                                                           next_protocol,
1179                                                           error);
1180                         if (ret < 0)
1181                                 return ret;
1182                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
1183                                                MLX5_FLOW_LAYER_OUTER_L4_UDP;
1184                         break;
1185                 case RTE_FLOW_ITEM_TYPE_TCP:
1186                         ret = mlx5_flow_validate_item_tcp
1187                                                 (items, item_flags,
1188                                                  next_protocol,
1189                                                  &rte_flow_item_tcp_mask,
1190                                                  error);
1191                         if (ret < 0)
1192                                 return ret;
1193                         item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
1194                                                MLX5_FLOW_LAYER_OUTER_L4_TCP;
1195                         break;
1196                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1197                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
1198                                                             error);
1199                         if (ret < 0)
1200                                 return ret;
1201                         item_flags |= MLX5_FLOW_LAYER_VXLAN;
1202                         break;
1203                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1204                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
1205                                                                 item_flags,
1206                                                                 dev, error);
1207                         if (ret < 0)
1208                                 return ret;
1209                         item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
1210                         break;
1211                 case RTE_FLOW_ITEM_TYPE_GRE:
1212                         ret = mlx5_flow_validate_item_gre(items, item_flags,
1213                                                           next_protocol, error);
1214                         if (ret < 0)
1215                                 return ret;
1216                         item_flags |= MLX5_FLOW_LAYER_GRE;
1217                         break;
1218                 case RTE_FLOW_ITEM_TYPE_MPLS:
1219                         ret = mlx5_flow_validate_item_mpls(items, item_flags,
1220                                                            next_protocol,
1221                                                            error);
1222                         if (ret < 0)
1223                                 return ret;
1224                         if (next_protocol != 0xff &&
1225                             next_protocol != IPPROTO_MPLS)
1226                                 return rte_flow_error_set
1227                                         (error, EINVAL,
1228                                          RTE_FLOW_ERROR_TYPE_ITEM, items,
1229                                          "protocol filtering not compatible"
1230                                          " with MPLS layer");
1231                         item_flags |= MLX5_FLOW_LAYER_MPLS;
1232                         break;
1233                 default:
1234                         return rte_flow_error_set(error, ENOTSUP,
1235                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1236                                                   NULL, "item not supported");
1237                 }
1238         }
1239         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1240                 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1241                 switch (actions->type) {
1242                 case RTE_FLOW_ACTION_TYPE_VOID:
1243                         break;
1244                 case RTE_FLOW_ACTION_TYPE_FLAG:
1245                         ret = mlx5_flow_validate_action_flag(action_flags,
1246                                                              attr,
1247                                                              error);
1248                         if (ret < 0)
1249                                 return ret;
1250                         action_flags |= MLX5_FLOW_ACTION_FLAG;
1251                         break;
1252                 case RTE_FLOW_ACTION_TYPE_MARK:
1253                         ret = mlx5_flow_validate_action_mark(actions,
1254                                                              action_flags,
1255                                                              attr,
1256                                                              error);
1257                         if (ret < 0)
1258                                 return ret;
1259                         action_flags |= MLX5_FLOW_ACTION_MARK;
1260                         break;
1261                 case RTE_FLOW_ACTION_TYPE_DROP:
1262                         ret = mlx5_flow_validate_action_drop(action_flags,
1263                                                              attr,
1264                                                              error);
1265                         if (ret < 0)
1266                                 return ret;
1267                         action_flags |= MLX5_FLOW_ACTION_DROP;
1268                         break;
1269                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1270                         ret = mlx5_flow_validate_action_queue(actions,
1271                                                               action_flags, dev,
1272                                                               attr,
1273                                                               error);
1274                         if (ret < 0)
1275                                 return ret;
1276                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
1277                         break;
1278                 case RTE_FLOW_ACTION_TYPE_RSS:
1279                         ret = mlx5_flow_validate_action_rss(actions,
1280                                                             action_flags, dev,
1281                                                             attr,
1282                                                             error);
1283                         if (ret < 0)
1284                                 return ret;
1285                         action_flags |= MLX5_FLOW_ACTION_RSS;
1286                         break;
1287                 case RTE_FLOW_ACTION_TYPE_COUNT:
1288                         ret = mlx5_flow_validate_action_count(dev, attr, error);
1289                         if (ret < 0)
1290                                 return ret;
1291                         action_flags |= MLX5_FLOW_ACTION_COUNT;
1292                         break;
1293                 default:
1294                         return rte_flow_error_set(error, ENOTSUP,
1295                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1296                                                   actions,
1297                                                   "action not supported");
1298                 }
1299         }
1300         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS))
1301                 return rte_flow_error_set(error, EINVAL,
1302                                           RTE_FLOW_ERROR_TYPE_ACTION, actions,
1303                                           "no fate action is found");
1304         return 0;
1305 }
1306
1307 /**
1308  * Calculate the required bytes that are needed for the action part of the verbs
1309  * flow, in addtion returns bit-fields with all the detected action, in order to
1310  * avoid another interation over the actions.
1311  *
1312  * @param[in] actions
1313  *   Pointer to the list of actions.
1314  * @param[out] action_flags
1315  *   Pointer to the detected actions.
1316  *
1317  * @return
1318  *   The size of the memory needed for all actions.
1319  */
1320 static int
1321 flow_verbs_get_actions_and_size(const struct rte_flow_action actions[],
1322                                 uint64_t *action_flags)
1323 {
1324         int size = 0;
1325         uint64_t detected_actions = 0;
1326
1327         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1328                 switch (actions->type) {
1329                 case RTE_FLOW_ACTION_TYPE_VOID:
1330                         break;
1331                 case RTE_FLOW_ACTION_TYPE_FLAG:
1332                         size += sizeof(struct ibv_flow_spec_action_tag);
1333                         detected_actions |= MLX5_FLOW_ACTION_FLAG;
1334                         break;
1335                 case RTE_FLOW_ACTION_TYPE_MARK:
1336                         size += sizeof(struct ibv_flow_spec_action_tag);
1337                         detected_actions |= MLX5_FLOW_ACTION_MARK;
1338                         break;
1339                 case RTE_FLOW_ACTION_TYPE_DROP:
1340                         size += sizeof(struct ibv_flow_spec_action_drop);
1341                         detected_actions |= MLX5_FLOW_ACTION_DROP;
1342                         break;
1343                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1344                         detected_actions |= MLX5_FLOW_ACTION_QUEUE;
1345                         break;
1346                 case RTE_FLOW_ACTION_TYPE_RSS:
1347                         detected_actions |= MLX5_FLOW_ACTION_RSS;
1348                         break;
1349                 case RTE_FLOW_ACTION_TYPE_COUNT:
1350 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1351         defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1352                         size += sizeof(struct ibv_flow_spec_counter_action);
1353 #endif
1354                         detected_actions |= MLX5_FLOW_ACTION_COUNT;
1355                         break;
1356                 default:
1357                         break;
1358                 }
1359         }
1360         *action_flags = detected_actions;
1361         return size;
1362 }
1363
1364 /**
1365  * Calculate the required bytes that are needed for the item part of the verbs
1366  * flow, in addtion returns bit-fields with all the detected action, in order to
1367  * avoid another interation over the actions.
1368  *
1369  * @param[in] actions
1370  *   Pointer to the list of items.
1371  * @param[in, out] item_flags
1372  *   Pointer to the detected items.
1373  *
1374  * @return
1375  *   The size of the memory needed for all items.
1376  */
1377 static int
1378 flow_verbs_get_items_and_size(const struct rte_flow_item items[],
1379                               uint64_t *item_flags)
1380 {
1381         int size = 0;
1382         uint64_t detected_items = 0;
1383         const int tunnel = !!(*item_flags & MLX5_FLOW_LAYER_TUNNEL);
1384
1385         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1386                 switch (items->type) {
1387                 case RTE_FLOW_ITEM_TYPE_VOID:
1388                         break;
1389                 case RTE_FLOW_ITEM_TYPE_ETH:
1390                         size += sizeof(struct ibv_flow_spec_eth);
1391                         detected_items |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
1392                                                    MLX5_FLOW_LAYER_OUTER_L2;
1393                         break;
1394                 case RTE_FLOW_ITEM_TYPE_VLAN:
1395                         size += sizeof(struct ibv_flow_spec_eth);
1396                         detected_items |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1397                                                    MLX5_FLOW_LAYER_OUTER_VLAN;
1398                         break;
1399                 case RTE_FLOW_ITEM_TYPE_IPV4:
1400                         size += sizeof(struct ibv_flow_spec_ipv4_ext);
1401                         detected_items |= tunnel ?
1402                                           MLX5_FLOW_LAYER_INNER_L3_IPV4 :
1403                                           MLX5_FLOW_LAYER_OUTER_L3_IPV4;
1404                         break;
1405                 case RTE_FLOW_ITEM_TYPE_IPV6:
1406                         size += sizeof(struct ibv_flow_spec_ipv6);
1407                         detected_items |= tunnel ?
1408                                           MLX5_FLOW_LAYER_INNER_L3_IPV6 :
1409                                           MLX5_FLOW_LAYER_OUTER_L3_IPV6;
1410                         break;
1411                 case RTE_FLOW_ITEM_TYPE_UDP:
1412                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1413                         detected_items |= tunnel ?
1414                                           MLX5_FLOW_LAYER_INNER_L4_UDP :
1415                                           MLX5_FLOW_LAYER_OUTER_L4_UDP;
1416                         break;
1417                 case RTE_FLOW_ITEM_TYPE_TCP:
1418                         size += sizeof(struct ibv_flow_spec_tcp_udp);
1419                         detected_items |= tunnel ?
1420                                           MLX5_FLOW_LAYER_INNER_L4_TCP :
1421                                           MLX5_FLOW_LAYER_OUTER_L4_TCP;
1422                         break;
1423                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1424                         size += sizeof(struct ibv_flow_spec_tunnel);
1425                         detected_items |= MLX5_FLOW_LAYER_VXLAN;
1426                         break;
1427                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1428                         size += sizeof(struct ibv_flow_spec_tunnel);
1429                         detected_items |= MLX5_FLOW_LAYER_VXLAN_GPE;
1430                         break;
1431 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1432                 case RTE_FLOW_ITEM_TYPE_GRE:
1433                         size += sizeof(struct ibv_flow_spec_gre);
1434                         detected_items |= MLX5_FLOW_LAYER_GRE;
1435                         break;
1436                 case RTE_FLOW_ITEM_TYPE_MPLS:
1437                         size += sizeof(struct ibv_flow_spec_mpls);
1438                         detected_items |= MLX5_FLOW_LAYER_MPLS;
1439                         break;
1440 #else
1441                 case RTE_FLOW_ITEM_TYPE_GRE:
1442                         size += sizeof(struct ibv_flow_spec_tunnel);
1443                         detected_items |= MLX5_FLOW_LAYER_TUNNEL;
1444                         break;
1445 #endif
1446                 default:
1447                         break;
1448                 }
1449         }
1450         *item_flags = detected_items;
1451         return size;
1452 }
1453
1454 /**
1455  * Internal preparation function. Allocate mlx5_flow with the required size.
1456  * The required size is calculate based on the actions and items. This function
1457  * also returns the detected actions and items for later use.
1458  *
1459  * @param[in] attr
1460  *   Pointer to the flow attributes.
1461  * @param[in] items
1462  *   Pointer to the list of items.
1463  * @param[in] actions
1464  *   Pointer to the list of actions.
1465  * @param[out] item_flags
1466  *   Pointer to bit mask of all items detected.
1467  * @param[out] action_flags
1468  *   Pointer to bit mask of all actions detected.
1469  * @param[out] error
1470  *   Pointer to the error structure.
1471  *
1472  * @return
1473  *   Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1474  *   is set.
1475  */
1476 static struct mlx5_flow *
1477 flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
1478                    const struct rte_flow_item items[],
1479                    const struct rte_flow_action actions[],
1480                    uint64_t *item_flags,
1481                    uint64_t *action_flags,
1482                    struct rte_flow_error *error)
1483 {
1484         uint32_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
1485         struct mlx5_flow *flow;
1486
1487         size += flow_verbs_get_actions_and_size(actions, action_flags);
1488         size += flow_verbs_get_items_and_size(items, item_flags);
1489         flow = rte_calloc(__func__, 1, size, 0);
1490         if (!flow) {
1491                 rte_flow_error_set(error, ENOMEM,
1492                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1493                                    "not enough memory to create flow");
1494                 return NULL;
1495         }
1496         flow->verbs.attr = (void *)(flow + 1);
1497         flow->verbs.specs =
1498                 (uint8_t *)(flow + 1) + sizeof(struct ibv_flow_attr);
1499         return flow;
1500 }
1501
1502 /**
1503  * Fill the flow with verb spec.
1504  *
1505  * @param[in] dev
1506  *   Pointer to Ethernet device.
1507  * @param[in, out] dev_flow
1508  *   Pointer to the mlx5 flow.
1509  * @param[in] attr
1510  *   Pointer to the flow attributes.
1511  * @param[in] items
1512  *   Pointer to the list of items.
1513  * @param[in] actions
1514  *   Pointer to the list of actions.
1515  * @param[out] error
1516  *   Pointer to the error structure.
1517  *
1518  * @return
1519  *   0 on success, else a negative errno value otherwise and rte_ernno is set.
1520  */
1521 static int
1522 flow_verbs_translate(struct rte_eth_dev *dev,
1523                      struct mlx5_flow *dev_flow,
1524                      const struct rte_flow_attr *attr,
1525                      const struct rte_flow_item items[],
1526                      const struct rte_flow_action actions[],
1527                      struct rte_flow_error *error)
1528 {
1529         uint64_t action_flags = 0;
1530         uint64_t item_flags = 0;
1531         uint64_t priority = attr->priority;
1532         struct priv *priv = dev->data->dev_private;
1533
1534         if (priority == MLX5_FLOW_PRIO_RSVD)
1535                 priority = priv->config.flow_prio - 1;
1536         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1537                 int ret;
1538                 switch (actions->type) {
1539                 case RTE_FLOW_ACTION_TYPE_VOID:
1540                         break;
1541                 case RTE_FLOW_ACTION_TYPE_FLAG:
1542                         flow_verbs_translate_action_flag(actions,
1543                                                          &action_flags,
1544                                                          dev_flow);
1545                         break;
1546                 case RTE_FLOW_ACTION_TYPE_MARK:
1547                         flow_verbs_translate_action_mark(actions,
1548                                                          &action_flags,
1549                                                          dev_flow);
1550                         break;
1551                 case RTE_FLOW_ACTION_TYPE_DROP:
1552                         flow_verbs_translate_action_drop(&action_flags,
1553                                                          dev_flow);
1554                         break;
1555                 case RTE_FLOW_ACTION_TYPE_QUEUE:
1556                         flow_verbs_translate_action_queue(actions,
1557                                                           &action_flags,
1558                                                           dev_flow);
1559                         break;
1560                 case RTE_FLOW_ACTION_TYPE_RSS:
1561                         flow_verbs_translate_action_rss(actions,
1562                                                         &action_flags,
1563                                                         dev_flow);
1564                         break;
1565                 case RTE_FLOW_ACTION_TYPE_COUNT:
1566                         ret = flow_verbs_translate_action_count(dev,
1567                                                                 actions,
1568                                                                 &action_flags,
1569                                                                 dev_flow,
1570                                                                 error);
1571                         if (ret < 0)
1572                                 return ret;
1573                         break;
1574                 default:
1575                         return rte_flow_error_set(error, ENOTSUP,
1576                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1577                                                   actions,
1578                                                   "action not supported");
1579                 }
1580         }
1581         dev_flow->flow->actions |= action_flags;
1582         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1583                 switch (items->type) {
1584                 case RTE_FLOW_ITEM_TYPE_VOID:
1585                         break;
1586                 case RTE_FLOW_ITEM_TYPE_ETH:
1587                         flow_verbs_translate_item_eth(items, &item_flags,
1588                                                       dev_flow);
1589                         break;
1590                 case RTE_FLOW_ITEM_TYPE_VLAN:
1591                         flow_verbs_translate_item_vlan(items, &item_flags,
1592                                                        dev_flow);
1593                         break;
1594                 case RTE_FLOW_ITEM_TYPE_IPV4:
1595                         flow_verbs_translate_item_ipv4(items, &item_flags,
1596                                                        dev_flow);
1597                         break;
1598                 case RTE_FLOW_ITEM_TYPE_IPV6:
1599                         flow_verbs_translate_item_ipv6(items, &item_flags,
1600                                                        dev_flow);
1601                         break;
1602                 case RTE_FLOW_ITEM_TYPE_UDP:
1603                         flow_verbs_translate_item_udp(items, &item_flags,
1604                                                       dev_flow);
1605                         break;
1606                 case RTE_FLOW_ITEM_TYPE_TCP:
1607                         flow_verbs_translate_item_tcp(items, &item_flags,
1608                                                       dev_flow);
1609                         break;
1610                 case RTE_FLOW_ITEM_TYPE_VXLAN:
1611                         flow_verbs_translate_item_vxlan(items, &item_flags,
1612                                                         dev_flow);
1613                         break;
1614                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1615                         flow_verbs_translate_item_vxlan_gpe(items, &item_flags,
1616                                                             dev_flow);
1617                         break;
1618                 case RTE_FLOW_ITEM_TYPE_GRE:
1619                         flow_verbs_translate_item_gre(items, &item_flags,
1620                                                       dev_flow);
1621                         break;
1622                 case RTE_FLOW_ITEM_TYPE_MPLS:
1623                         flow_verbs_translate_item_mpls(items, &item_flags,
1624                                                        dev_flow);
1625                         break;
1626                 default:
1627                         return rte_flow_error_set(error, ENOTSUP,
1628                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1629                                                   NULL,
1630                                                   "item not supported");
1631                 }
1632         }
1633         dev_flow->verbs.attr->priority =
1634                 mlx5_flow_adjust_priority(dev, priority,
1635                                           dev_flow->verbs.attr->priority);
1636         return 0;
1637 }
1638
1639 /**
1640  * Remove the flow from the NIC but keeps it in memory.
1641  *
1642  * @param[in] dev
1643  *   Pointer to the Ethernet device structure.
1644  * @param[in, out] flow
1645  *   Pointer to flow structure.
1646  */
1647 static void
1648 flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1649 {
1650         struct mlx5_flow_verbs *verbs;
1651         struct mlx5_flow *dev_flow;
1652
1653         if (!flow)
1654                 return;
1655         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1656                 verbs = &dev_flow->verbs;
1657                 if (verbs->flow) {
1658                         claim_zero(mlx5_glue->destroy_flow(verbs->flow));
1659                         verbs->flow = NULL;
1660                 }
1661                 if (verbs->hrxq) {
1662                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1663                                 mlx5_hrxq_drop_release(dev);
1664                         else
1665                                 mlx5_hrxq_release(dev, verbs->hrxq);
1666                         verbs->hrxq = NULL;
1667                 }
1668         }
1669         if (flow->counter) {
1670                 flow_verbs_counter_release(flow->counter);
1671                 flow->counter = NULL;
1672         }
1673 }
1674
1675 /**
1676  * Remove the flow from the NIC and the memory.
1677  *
1678  * @param[in] dev
1679  *   Pointer to the Ethernet device structure.
1680  * @param[in, out] flow
1681  *   Pointer to flow structure.
1682  */
1683 static void
1684 flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1685 {
1686         struct mlx5_flow *dev_flow;
1687
1688         if (!flow)
1689                 return;
1690         flow_verbs_remove(dev, flow);
1691         while (!LIST_EMPTY(&flow->dev_flows)) {
1692                 dev_flow = LIST_FIRST(&flow->dev_flows);
1693                 LIST_REMOVE(dev_flow, next);
1694                 rte_free(dev_flow);
1695         }
1696 }
1697
1698 /**
1699  * Apply the flow to the NIC.
1700  *
1701  * @param[in] dev
1702  *   Pointer to the Ethernet device structure.
1703  * @param[in, out] flow
1704  *   Pointer to flow structure.
1705  * @param[out] error
1706  *   Pointer to error structure.
1707  *
1708  * @return
1709  *   0 on success, a negative errno value otherwise and rte_errno is set.
1710  */
1711 static int
1712 flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1713                  struct rte_flow_error *error)
1714 {
1715         struct mlx5_flow_verbs *verbs;
1716         struct mlx5_flow *dev_flow;
1717         int err;
1718
1719         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1720                 verbs = &dev_flow->verbs;
1721                 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1722                         verbs->hrxq = mlx5_hrxq_drop_new(dev);
1723                         if (!verbs->hrxq) {
1724                                 rte_flow_error_set
1725                                         (error, errno,
1726                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1727                                          "cannot get drop hash queue");
1728                                 goto error;
1729                         }
1730                 } else {
1731                         struct mlx5_hrxq *hrxq;
1732
1733                         hrxq = mlx5_hrxq_get(dev, flow->key,
1734                                              MLX5_RSS_HASH_KEY_LEN,
1735                                              verbs->hash_fields,
1736                                              (*flow->queue),
1737                                              flow->rss.queue_num);
1738                         if (!hrxq)
1739                                 hrxq = mlx5_hrxq_new(dev, flow->key,
1740                                                      MLX5_RSS_HASH_KEY_LEN,
1741                                                      verbs->hash_fields,
1742                                                      (*flow->queue),
1743                                                      flow->rss.queue_num,
1744                                                      !!(flow->layers &
1745                                                       MLX5_FLOW_LAYER_TUNNEL));
1746                         if (!hrxq) {
1747                                 rte_flow_error_set
1748                                         (error, rte_errno,
1749                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1750                                          "cannot get hash queue");
1751                                 goto error;
1752                         }
1753                         verbs->hrxq = hrxq;
1754                 }
1755                 verbs->flow = mlx5_glue->create_flow(verbs->hrxq->qp,
1756                                                      verbs->attr);
1757                 if (!verbs->flow) {
1758                         rte_flow_error_set(error, errno,
1759                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1760                                            NULL,
1761                                            "hardware refuses to create flow");
1762                         goto error;
1763                 }
1764         }
1765         return 0;
1766 error:
1767         err = rte_errno; /* Save rte_errno before cleanup. */
1768         LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1769                 verbs = &dev_flow->verbs;
1770                 if (verbs->hrxq) {
1771                         if (flow->actions & MLX5_FLOW_ACTION_DROP)
1772                                 mlx5_hrxq_drop_release(dev);
1773                         else
1774                                 mlx5_hrxq_release(dev, verbs->hrxq);
1775                         verbs->hrxq = NULL;
1776                 }
1777         }
1778         rte_errno = err; /* Restore rte_errno. */
1779         return -rte_errno;
1780 }
1781
1782 /**
1783  * Query a flow.
1784  *
1785  * @see rte_flow_query()
1786  * @see rte_flow_ops
1787  */
1788 static int
1789 flow_verbs_query(struct rte_eth_dev *dev,
1790                  struct rte_flow *flow,
1791                  const struct rte_flow_action *actions,
1792                  void *data,
1793                  struct rte_flow_error *error)
1794 {
1795         int ret = -EINVAL;
1796
1797         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1798                 switch (actions->type) {
1799                 case RTE_FLOW_ACTION_TYPE_VOID:
1800                         break;
1801                 case RTE_FLOW_ACTION_TYPE_COUNT:
1802                         ret = flow_verbs_counter_query(dev, flow, data, error);
1803                         break;
1804                 default:
1805                         return rte_flow_error_set(error, ENOTSUP,
1806                                                   RTE_FLOW_ERROR_TYPE_ACTION,
1807                                                   actions,
1808                                                   "action not supported");
1809                 }
1810         }
1811         return ret;
1812 }
1813
1814 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
1815         .validate = flow_verbs_validate,
1816         .prepare = flow_verbs_prepare,
1817         .translate = flow_verbs_translate,
1818         .apply = flow_verbs_apply,
1819         .remove = flow_verbs_remove,
1820         .destroy = flow_verbs_destroy,
1821         .query = flow_verbs_query,
1822 };