drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_cycles.h>
  28 #include <rte_ip.h>
  29 #include <rte_gre.h>
  30 #include <rte_vxlan.h>
  31 #include <rte_gtp.h>
  32
  33 #include <mlx5_devx_cmds.h>
  34 #include <mlx5_prm.h>
  35
  36 #include "mlx5_defs.h"
  37 #include "mlx5.h"
  38 #include "mlx5_common_os.h"
  39 #include "mlx5_flow.h"
  40 #include "mlx5_flow_os.h"
  41 #include "mlx5_rxtx.h"
  42
  43 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  44
  45 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  46 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR_ESWITCH
  50 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  51 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  52 #endif
  53 #endif
  54
  55 #ifndef HAVE_MLX5DV_DR
  56 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  57 #endif
  58
  59 /* VLAN header definitions */
  60 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  61 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  62 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  63 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  64 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  65
  66 union flow_dv_attr {
  67         struct {
  68                 uint32_t valid:1;
  69                 uint32_t ipv4:1;
  70                 uint32_t ipv6:1;
  71                 uint32_t tcp:1;
  72                 uint32_t udp:1;
  73                 uint32_t reserved:27;
  74         };
  75         uint32_t attr;
  76 };
  77
  78 static int
  79 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  80                              struct mlx5_flow_tbl_resource *tbl);
  81
  82 static int
  83 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  84
  85 /**
  86  * Initialize flow attributes structure according to flow items' types.
  87  *
  88  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  89  * mode. For tunnel mode, the items to be modified are the outermost ones.
  90  *
  91  * @param[in] item
  92  *   Pointer to item specification.
  93  * @param[out] attr
  94  *   Pointer to flow attributes structure.
  95  * @param[in] dev_flow
  96  *   Pointer to the sub flow.
  97  * @param[in] tunnel_decap
  98  *   Whether action is after tunnel decapsulation.
  99  */
 100 static void
 101 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 102                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 103 {
 104         uint64_t layers = dev_flow->handle->layers;
 105
 106         /*
 107          * If layers is already initialized, it means this dev_flow is the
 108          * suffix flow, the layers flags is set by the prefix flow. Need to
 109          * use the layer flags from prefix flow as the suffix flow may not
 110          * have the user defined items as the flow is split.
 111          */
 112         if (layers) {
 113                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 114                         attr->ipv4 = 1;
 115                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 116                         attr->ipv6 = 1;
 117                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 118                         attr->tcp = 1;
 119                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 120                         attr->udp = 1;
 121                 attr->valid = 1;
 122                 return;
 123         }
 124         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 125                 uint8_t next_protocol = 0xff;
 126                 switch (item->type) {
 127                 case RTE_FLOW_ITEM_TYPE_GRE:
 128                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 129                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 130                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 131                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 132                 case RTE_FLOW_ITEM_TYPE_MPLS:
 133                         if (tunnel_decap)
 134                                 attr->attr = 0;
 135                         break;
 136                 case RTE_FLOW_ITEM_TYPE_IPV4:
 137                         if (!attr->ipv6)
 138                                 attr->ipv4 = 1;
 139                         if (item->mask != NULL &&
 140                             ((const struct rte_flow_item_ipv4 *)
 141                             item->mask)->hdr.next_proto_id)
 142                                 next_protocol =
 143                                     ((const struct rte_flow_item_ipv4 *)
 144                                       (item->spec))->hdr.next_proto_id &
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->mask))->hdr.next_proto_id;
 147                         if ((next_protocol == IPPROTO_IPIP ||
 148                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 149                                 attr->attr = 0;
 150                         break;
 151                 case RTE_FLOW_ITEM_TYPE_IPV6:
 152                         if (!attr->ipv4)
 153                                 attr->ipv6 = 1;
 154                         if (item->mask != NULL &&
 155                             ((const struct rte_flow_item_ipv6 *)
 156                             item->mask)->hdr.proto)
 157                                 next_protocol =
 158                                     ((const struct rte_flow_item_ipv6 *)
 159                                       (item->spec))->hdr.proto &
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->mask))->hdr.proto;
 162                         if ((next_protocol == IPPROTO_IPIP ||
 163                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 164                                 attr->attr = 0;
 165                         break;
 166                 case RTE_FLOW_ITEM_TYPE_UDP:
 167                         if (!attr->tcp)
 168                                 attr->udp = 1;
 169                         break;
 170                 case RTE_FLOW_ITEM_TYPE_TCP:
 171                         if (!attr->udp)
 172                                 attr->tcp = 1;
 173                         break;
 174                 default:
 175                         break;
 176                 }
 177         }
 178         attr->valid = 1;
 179 }
 180
 181 /**
 182  * Convert rte_mtr_color to mlx5 color.
 183  *
 184  * @param[in] rcol
 185  *   rte_mtr_color.
 186  *
 187  * @return
 188  *   mlx5 color.
 189  */
 190 static int
 191 rte_col_2_mlx5_col(enum rte_color rcol)
 192 {
 193         switch (rcol) {
 194         case RTE_COLOR_GREEN:
 195                 return MLX5_FLOW_COLOR_GREEN;
 196         case RTE_COLOR_YELLOW:
 197                 return MLX5_FLOW_COLOR_YELLOW;
 198         case RTE_COLOR_RED:
 199                 return MLX5_FLOW_COLOR_RED;
 200         default:
 201                 break;
 202         }
 203         return MLX5_FLOW_COLOR_UNDEFINED;
 204 }
 205
 206 struct field_modify_info {
 207         uint32_t size; /* Size of field in protocol header, in bytes. */
 208         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 209         enum mlx5_modification_field id;
 210 };
 211
 212 struct field_modify_info modify_eth[] = {
 213         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 214         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 215         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 216         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 217         {0, 0, 0},
 218 };
 219
 220 struct field_modify_info modify_vlan_out_first_vid[] = {
 221         /* Size in bits !!! */
 222         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 223         {0, 0, 0},
 224 };
 225
 226 struct field_modify_info modify_ipv4[] = {
 227         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 228         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 229         {4, 12, MLX5_MODI_OUT_SIPV4},
 230         {4, 16, MLX5_MODI_OUT_DIPV4},
 231         {0, 0, 0},
 232 };
 233
 234 struct field_modify_info modify_ipv6[] = {
 235         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 236         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 237         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 238         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 239         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 240         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 241         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 242         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 243         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 244         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 245         {0, 0, 0},
 246 };
 247
 248 struct field_modify_info modify_udp[] = {
 249         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 250         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 251         {0, 0, 0},
 252 };
 253
 254 struct field_modify_info modify_tcp[] = {
 255         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 256         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 257         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 258         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 259         {0, 0, 0},
 260 };
 261
 262 static void
 263 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 264                           uint8_t next_protocol, uint64_t *item_flags,
 265                           int *tunnel)
 266 {
 267         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 268                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 269         if (next_protocol == IPPROTO_IPIP) {
 270                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 271                 *tunnel = 1;
 272         }
 273         if (next_protocol == IPPROTO_IPV6) {
 274                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 275                 *tunnel = 1;
 276         }
 277 }
 278
 279 /**
 280  * Acquire the synchronizing object to protect multithreaded access
 281  * to shared dv context. Lock occurs only if context is actually
 282  * shared, i.e. we have multiport IB device and representors are
 283  * created.
 284  *
 285  * @param[in] dev
 286  *   Pointer to the rte_eth_dev structure.
 287  */
 288 static void
 289 flow_dv_shared_lock(struct rte_eth_dev *dev)
 290 {
 291         struct mlx5_priv *priv = dev->data->dev_private;
 292         struct mlx5_dev_ctx_shared *sh = priv->sh;
 293
 294         if (sh->dv_refcnt > 1) {
 295                 int ret;
 296
 297                 ret = pthread_mutex_lock(&sh->dv_mutex);
 298                 MLX5_ASSERT(!ret);
 299                 (void)ret;
 300         }
 301 }
 302
 303 static void
 304 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 305 {
 306         struct mlx5_priv *priv = dev->data->dev_private;
 307         struct mlx5_dev_ctx_shared *sh = priv->sh;
 308
 309         if (sh->dv_refcnt > 1) {
 310                 int ret;
 311
 312                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 313                 MLX5_ASSERT(!ret);
 314                 (void)ret;
 315         }
 316 }
 317
 318 /* Update VLAN's VID/PCP based on input rte_flow_action.
 319  *
 320  * @param[in] action
 321  *   Pointer to struct rte_flow_action.
 322  * @param[out] vlan
 323  *   Pointer to struct rte_vlan_hdr.
 324  */
 325 static void
 326 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 327                          struct rte_vlan_hdr *vlan)
 328 {
 329         uint16_t vlan_tci;
 330         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 331                 vlan_tci =
 332                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 333                                                action->conf)->vlan_pcp;
 334                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 335                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 336                 vlan->vlan_tci |= vlan_tci;
 337         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 338                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 339                 vlan->vlan_tci |= rte_be_to_cpu_16
 340                     (((const struct rte_flow_action_of_set_vlan_vid *)
 341                                              action->conf)->vlan_vid);
 342         }
 343 }
 344
 345 /**
 346  * Fetch 1, 2, 3 or 4 byte field from the byte array
 347  * and return as unsigned integer in host-endian format.
 348  *
 349  * @param[in] data
 350  *   Pointer to data array.
 351  * @param[in] size
 352  *   Size of field to extract.
 353  *
 354  * @return
 355  *   converted field in host endian format.
 356  */
 357 static inline uint32_t
 358 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 359 {
 360         uint32_t ret;
 361
 362         switch (size) {
 363         case 1:
 364                 ret = *data;
 365                 break;
 366         case 2:
 367                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 368                 break;
 369         case 3:
 370                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 371                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 372                 break;
 373         case 4:
 374                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 375                 break;
 376         default:
 377                 MLX5_ASSERT(false);
 378                 ret = 0;
 379                 break;
 380         }
 381         return ret;
 382 }
 383
 384 /**
 385  * Convert modify-header action to DV specification.
 386  *
 387  * Data length of each action is determined by provided field description
 388  * and the item mask. Data bit offset and width of each action is determined
 389  * by provided item mask.
 390  *
 391  * @param[in] item
 392  *   Pointer to item specification.
 393  * @param[in] field
 394  *   Pointer to field modification information.
 395  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 396  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 397  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 398  * @param[in] dcopy
 399  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 400  *   Negative offset value sets the same offset as source offset.
 401  *   size field is ignored, value is taken from source field.
 402  * @param[in,out] resource
 403  *   Pointer to the modify-header resource.
 404  * @param[in] type
 405  *   Type of modification.
 406  * @param[out] error
 407  *   Pointer to the error structure.
 408  *
 409  * @return
 410  *   0 on success, a negative errno value otherwise and rte_errno is set.
 411  */
 412 static int
 413 flow_dv_convert_modify_action(struct rte_flow_item *item,
 414                               struct field_modify_info *field,
 415                               struct field_modify_info *dcopy,
 416                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 417                               uint32_t type, struct rte_flow_error *error)
 418 {
 419         uint32_t i = resource->actions_num;
 420         struct mlx5_modification_cmd *actions = resource->actions;
 421
 422         /*
 423          * The item and mask are provided in big-endian format.
 424          * The fields should be presented as in big-endian format either.
 425          * Mask must be always present, it defines the actual field width.
 426          */
 427         MLX5_ASSERT(item->mask);
 428         MLX5_ASSERT(field->size);
 429         do {
 430                 unsigned int size_b;
 431                 unsigned int off_b;
 432                 uint32_t mask;
 433                 uint32_t data;
 434
 435                 if (i >= MLX5_MAX_MODIFY_NUM)
 436                         return rte_flow_error_set(error, EINVAL,
 437                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 438                                  "too many items to modify");
 439                 /* Fetch variable byte size mask from the array. */
 440                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 441                                            field->offset, field->size);
 442                 if (!mask) {
 443                         ++field;
 444                         continue;
 445                 }
 446                 /* Deduce actual data width in bits from mask value. */
 447                 off_b = rte_bsf32(mask);
 448                 size_b = sizeof(uint32_t) * CHAR_BIT -
 449                          off_b - __builtin_clz(mask);
 450                 MLX5_ASSERT(size_b);
 451                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 452                 actions[i] = (struct mlx5_modification_cmd) {
 453                         .action_type = type,
 454                         .field = field->id,
 455                         .offset = off_b,
 456                         .length = size_b,
 457                 };
 458                 /* Convert entire record to expected big-endian format. */
 459                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 460                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 461                         MLX5_ASSERT(dcopy);
 462                         actions[i].dst_field = dcopy->id;
 463                         actions[i].dst_offset =
 464                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 465                         /* Convert entire record to big-endian format. */
 466                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 467                 } else {
 468                         MLX5_ASSERT(item->spec);
 469                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 470                                                    field->offset, field->size);
 471                         /* Shift out the trailing masked bits from data. */
 472                         data = (data & mask) >> off_b;
 473                         actions[i].data1 = rte_cpu_to_be_32(data);
 474                 }
 475                 ++i;
 476                 ++field;
 477         } while (field->size);
 478         if (resource->actions_num == i)
 479                 return rte_flow_error_set(error, EINVAL,
 480                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 481                                           "invalid modification flow item");
 482         resource->actions_num = i;
 483         return 0;
 484 }
 485
 486 /**
 487  * Convert modify-header set IPv4 address action to DV specification.
 488  *
 489  * @param[in,out] resource
 490  *   Pointer to the modify-header resource.
 491  * @param[in] action
 492  *   Pointer to action specification.
 493  * @param[out] error
 494  *   Pointer to the error structure.
 495  *
 496  * @return
 497  *   0 on success, a negative errno value otherwise and rte_errno is set.
 498  */
 499 static int
 500 flow_dv_convert_action_modify_ipv4
 501                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 502                          const struct rte_flow_action *action,
 503                          struct rte_flow_error *error)
 504 {
 505         const struct rte_flow_action_set_ipv4 *conf =
 506                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 507         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 508         struct rte_flow_item_ipv4 ipv4;
 509         struct rte_flow_item_ipv4 ipv4_mask;
 510
 511         memset(&ipv4, 0, sizeof(ipv4));
 512         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 513         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 514                 ipv4.hdr.src_addr = conf->ipv4_addr;
 515                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 516         } else {
 517                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 518                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 519         }
 520         item.spec = &ipv4;
 521         item.mask = &ipv4_mask;
 522         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 523                                              MLX5_MODIFICATION_TYPE_SET, error);
 524 }
 525
 526 /**
 527  * Convert modify-header set IPv6 address action to DV specification.
 528  *
 529  * @param[in,out] resource
 530  *   Pointer to the modify-header resource.
 531  * @param[in] action
 532  *   Pointer to action specification.
 533  * @param[out] error
 534  *   Pointer to the error structure.
 535  *
 536  * @return
 537  *   0 on success, a negative errno value otherwise and rte_errno is set.
 538  */
 539 static int
 540 flow_dv_convert_action_modify_ipv6
 541                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 542                          const struct rte_flow_action *action,
 543                          struct rte_flow_error *error)
 544 {
 545         const struct rte_flow_action_set_ipv6 *conf =
 546                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 547         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 548         struct rte_flow_item_ipv6 ipv6;
 549         struct rte_flow_item_ipv6 ipv6_mask;
 550
 551         memset(&ipv6, 0, sizeof(ipv6));
 552         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 553         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 554                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 555                        sizeof(ipv6.hdr.src_addr));
 556                 memcpy(&ipv6_mask.hdr.src_addr,
 557                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 558                        sizeof(ipv6.hdr.src_addr));
 559         } else {
 560                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 561                        sizeof(ipv6.hdr.dst_addr));
 562                 memcpy(&ipv6_mask.hdr.dst_addr,
 563                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 564                        sizeof(ipv6.hdr.dst_addr));
 565         }
 566         item.spec = &ipv6;
 567         item.mask = &ipv6_mask;
 568         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 569                                              MLX5_MODIFICATION_TYPE_SET, error);
 570 }
 571
 572 /**
 573  * Convert modify-header set MAC address action to DV specification.
 574  *
 575  * @param[in,out] resource
 576  *   Pointer to the modify-header resource.
 577  * @param[in] action
 578  *   Pointer to action specification.
 579  * @param[out] error
 580  *   Pointer to the error structure.
 581  *
 582  * @return
 583  *   0 on success, a negative errno value otherwise and rte_errno is set.
 584  */
 585 static int
 586 flow_dv_convert_action_modify_mac
 587                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 588                          const struct rte_flow_action *action,
 589                          struct rte_flow_error *error)
 590 {
 591         const struct rte_flow_action_set_mac *conf =
 592                 (const struct rte_flow_action_set_mac *)(action->conf);
 593         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 594         struct rte_flow_item_eth eth;
 595         struct rte_flow_item_eth eth_mask;
 596
 597         memset(&eth, 0, sizeof(eth));
 598         memset(&eth_mask, 0, sizeof(eth_mask));
 599         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 600                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 601                        sizeof(eth.src.addr_bytes));
 602                 memcpy(&eth_mask.src.addr_bytes,
 603                        &rte_flow_item_eth_mask.src.addr_bytes,
 604                        sizeof(eth_mask.src.addr_bytes));
 605         } else {
 606                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 607                        sizeof(eth.dst.addr_bytes));
 608                 memcpy(&eth_mask.dst.addr_bytes,
 609                        &rte_flow_item_eth_mask.dst.addr_bytes,
 610                        sizeof(eth_mask.dst.addr_bytes));
 611         }
 612         item.spec = &eth;
 613         item.mask = &eth_mask;
 614         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 615                                              MLX5_MODIFICATION_TYPE_SET, error);
 616 }
 617
 618 /**
 619  * Convert modify-header set VLAN VID action to DV specification.
 620  *
 621  * @param[in,out] resource
 622  *   Pointer to the modify-header resource.
 623  * @param[in] action
 624  *   Pointer to action specification.
 625  * @param[out] error
 626  *   Pointer to the error structure.
 627  *
 628  * @return
 629  *   0 on success, a negative errno value otherwise and rte_errno is set.
 630  */
 631 static int
 632 flow_dv_convert_action_modify_vlan_vid
 633                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 634                          const struct rte_flow_action *action,
 635                          struct rte_flow_error *error)
 636 {
 637         const struct rte_flow_action_of_set_vlan_vid *conf =
 638                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 639         int i = resource->actions_num;
 640         struct mlx5_modification_cmd *actions = resource->actions;
 641         struct field_modify_info *field = modify_vlan_out_first_vid;
 642
 643         if (i >= MLX5_MAX_MODIFY_NUM)
 644                 return rte_flow_error_set(error, EINVAL,
 645                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 646                          "too many items to modify");
 647         actions[i] = (struct mlx5_modification_cmd) {
 648                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 649                 .field = field->id,
 650                 .length = field->size,
 651                 .offset = field->offset,
 652         };
 653         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 654         actions[i].data1 = conf->vlan_vid;
 655         actions[i].data1 = actions[i].data1 << 16;
 656         resource->actions_num = ++i;
 657         return 0;
 658 }
 659
 660 /**
 661  * Convert modify-header set TP action to DV specification.
 662  *
 663  * @param[in,out] resource
 664  *   Pointer to the modify-header resource.
 665  * @param[in] action
 666  *   Pointer to action specification.
 667  * @param[in] items
 668  *   Pointer to rte_flow_item objects list.
 669  * @param[in] attr
 670  *   Pointer to flow attributes structure.
 671  * @param[in] dev_flow
 672  *   Pointer to the sub flow.
 673  * @param[in] tunnel_decap
 674  *   Whether action is after tunnel decapsulation.
 675  * @param[out] error
 676  *   Pointer to the error structure.
 677  *
 678  * @return
 679  *   0 on success, a negative errno value otherwise and rte_errno is set.
 680  */
 681 static int
 682 flow_dv_convert_action_modify_tp
 683                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 684                          const struct rte_flow_action *action,
 685                          const struct rte_flow_item *items,
 686                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 687                          bool tunnel_decap, struct rte_flow_error *error)
 688 {
 689         const struct rte_flow_action_set_tp *conf =
 690                 (const struct rte_flow_action_set_tp *)(action->conf);
 691         struct rte_flow_item item;
 692         struct rte_flow_item_udp udp;
 693         struct rte_flow_item_udp udp_mask;
 694         struct rte_flow_item_tcp tcp;
 695         struct rte_flow_item_tcp tcp_mask;
 696         struct field_modify_info *field;
 697
 698         if (!attr->valid)
 699                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 700         if (attr->udp) {
 701                 memset(&udp, 0, sizeof(udp));
 702                 memset(&udp_mask, 0, sizeof(udp_mask));
 703                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 704                         udp.hdr.src_port = conf->port;
 705                         udp_mask.hdr.src_port =
 706                                         rte_flow_item_udp_mask.hdr.src_port;
 707                 } else {
 708                         udp.hdr.dst_port = conf->port;
 709                         udp_mask.hdr.dst_port =
 710                                         rte_flow_item_udp_mask.hdr.dst_port;
 711                 }
 712                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 713                 item.spec = &udp;
 714                 item.mask = &udp_mask;
 715                 field = modify_udp;
 716         } else {
 717                 MLX5_ASSERT(attr->tcp);
 718                 memset(&tcp, 0, sizeof(tcp));
 719                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 720                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 721                         tcp.hdr.src_port = conf->port;
 722                         tcp_mask.hdr.src_port =
 723                                         rte_flow_item_tcp_mask.hdr.src_port;
 724                 } else {
 725                         tcp.hdr.dst_port = conf->port;
 726                         tcp_mask.hdr.dst_port =
 727                                         rte_flow_item_tcp_mask.hdr.dst_port;
 728                 }
 729                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 730                 item.spec = &tcp;
 731                 item.mask = &tcp_mask;
 732                 field = modify_tcp;
 733         }
 734         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 735                                              MLX5_MODIFICATION_TYPE_SET, error);
 736 }
 737
 738 /**
 739  * Convert modify-header set TTL action to DV specification.
 740  *
 741  * @param[in,out] resource
 742  *   Pointer to the modify-header resource.
 743  * @param[in] action
 744  *   Pointer to action specification.
 745  * @param[in] items
 746  *   Pointer to rte_flow_item objects list.
 747  * @param[in] attr
 748  *   Pointer to flow attributes structure.
 749  * @param[in] dev_flow
 750  *   Pointer to the sub flow.
 751  * @param[in] tunnel_decap
 752  *   Whether action is after tunnel decapsulation.
 753  * @param[out] error
 754  *   Pointer to the error structure.
 755  *
 756  * @return
 757  *   0 on success, a negative errno value otherwise and rte_errno is set.
 758  */
 759 static int
 760 flow_dv_convert_action_modify_ttl
 761                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 762                          const struct rte_flow_action *action,
 763                          const struct rte_flow_item *items,
 764                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 765                          bool tunnel_decap, struct rte_flow_error *error)
 766 {
 767         const struct rte_flow_action_set_ttl *conf =
 768                 (const struct rte_flow_action_set_ttl *)(action->conf);
 769         struct rte_flow_item item;
 770         struct rte_flow_item_ipv4 ipv4;
 771         struct rte_flow_item_ipv4 ipv4_mask;
 772         struct rte_flow_item_ipv6 ipv6;
 773         struct rte_flow_item_ipv6 ipv6_mask;
 774         struct field_modify_info *field;
 775
 776         if (!attr->valid)
 777                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 778         if (attr->ipv4) {
 779                 memset(&ipv4, 0, sizeof(ipv4));
 780                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 781                 ipv4.hdr.time_to_live = conf->ttl_value;
 782                 ipv4_mask.hdr.time_to_live = 0xFF;
 783                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 784                 item.spec = &ipv4;
 785                 item.mask = &ipv4_mask;
 786                 field = modify_ipv4;
 787         } else {
 788                 MLX5_ASSERT(attr->ipv6);
 789                 memset(&ipv6, 0, sizeof(ipv6));
 790                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 791                 ipv6.hdr.hop_limits = conf->ttl_value;
 792                 ipv6_mask.hdr.hop_limits = 0xFF;
 793                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 794                 item.spec = &ipv6;
 795                 item.mask = &ipv6_mask;
 796                 field = modify_ipv6;
 797         }
 798         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 799                                              MLX5_MODIFICATION_TYPE_SET, error);
 800 }
 801
 802 /**
 803  * Convert modify-header decrement TTL action to DV specification.
 804  *
 805  * @param[in,out] resource
 806  *   Pointer to the modify-header resource.
 807  * @param[in] action
 808  *   Pointer to action specification.
 809  * @param[in] items
 810  *   Pointer to rte_flow_item objects list.
 811  * @param[in] attr
 812  *   Pointer to flow attributes structure.
 813  * @param[in] dev_flow
 814  *   Pointer to the sub flow.
 815  * @param[in] tunnel_decap
 816  *   Whether action is after tunnel decapsulation.
 817  * @param[out] error
 818  *   Pointer to the error structure.
 819  *
 820  * @return
 821  *   0 on success, a negative errno value otherwise and rte_errno is set.
 822  */
 823 static int
 824 flow_dv_convert_action_modify_dec_ttl
 825                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 826                          const struct rte_flow_item *items,
 827                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 828                          bool tunnel_decap, struct rte_flow_error *error)
 829 {
 830         struct rte_flow_item item;
 831         struct rte_flow_item_ipv4 ipv4;
 832         struct rte_flow_item_ipv4 ipv4_mask;
 833         struct rte_flow_item_ipv6 ipv6;
 834         struct rte_flow_item_ipv6 ipv6_mask;
 835         struct field_modify_info *field;
 836
 837         if (!attr->valid)
 838                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 839         if (attr->ipv4) {
 840                 memset(&ipv4, 0, sizeof(ipv4));
 841                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 842                 ipv4.hdr.time_to_live = 0xFF;
 843                 ipv4_mask.hdr.time_to_live = 0xFF;
 844                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 845                 item.spec = &ipv4;
 846                 item.mask = &ipv4_mask;
 847                 field = modify_ipv4;
 848         } else {
 849                 MLX5_ASSERT(attr->ipv6);
 850                 memset(&ipv6, 0, sizeof(ipv6));
 851                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 852                 ipv6.hdr.hop_limits = 0xFF;
 853                 ipv6_mask.hdr.hop_limits = 0xFF;
 854                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 855                 item.spec = &ipv6;
 856                 item.mask = &ipv6_mask;
 857                 field = modify_ipv6;
 858         }
 859         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 860                                              MLX5_MODIFICATION_TYPE_ADD, error);
 861 }
 862
 863 /**
 864  * Convert modify-header increment/decrement TCP Sequence number
 865  * to DV specification.
 866  *
 867  * @param[in,out] resource
 868  *   Pointer to the modify-header resource.
 869  * @param[in] action
 870  *   Pointer to action specification.
 871  * @param[out] error
 872  *   Pointer to the error structure.
 873  *
 874  * @return
 875  *   0 on success, a negative errno value otherwise and rte_errno is set.
 876  */
 877 static int
 878 flow_dv_convert_action_modify_tcp_seq
 879                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 880                          const struct rte_flow_action *action,
 881                          struct rte_flow_error *error)
 882 {
 883         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 884         uint64_t value = rte_be_to_cpu_32(*conf);
 885         struct rte_flow_item item;
 886         struct rte_flow_item_tcp tcp;
 887         struct rte_flow_item_tcp tcp_mask;
 888
 889         memset(&tcp, 0, sizeof(tcp));
 890         memset(&tcp_mask, 0, sizeof(tcp_mask));
 891         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 892                 /*
 893                  * The HW has no decrement operation, only increment operation.
 894                  * To simulate decrement X from Y using increment operation
 895                  * we need to add UINT32_MAX X times to Y.
 896                  * Each adding of UINT32_MAX decrements Y by 1.
 897                  */
 898                 value *= UINT32_MAX;
 899         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 900         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 901         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 902         item.spec = &tcp;
 903         item.mask = &tcp_mask;
 904         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 905                                              MLX5_MODIFICATION_TYPE_ADD, error);
 906 }
 907
 908 /**
 909  * Convert modify-header increment/decrement TCP Acknowledgment number
 910  * to DV specification.
 911  *
 912  * @param[in,out] resource
 913  *   Pointer to the modify-header resource.
 914  * @param[in] action
 915  *   Pointer to action specification.
 916  * @param[out] error
 917  *   Pointer to the error structure.
 918  *
 919  * @return
 920  *   0 on success, a negative errno value otherwise and rte_errno is set.
 921  */
 922 static int
 923 flow_dv_convert_action_modify_tcp_ack
 924                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 925                          const struct rte_flow_action *action,
 926                          struct rte_flow_error *error)
 927 {
 928         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 929         uint64_t value = rte_be_to_cpu_32(*conf);
 930         struct rte_flow_item item;
 931         struct rte_flow_item_tcp tcp;
 932         struct rte_flow_item_tcp tcp_mask;
 933
 934         memset(&tcp, 0, sizeof(tcp));
 935         memset(&tcp_mask, 0, sizeof(tcp_mask));
 936         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 937                 /*
 938                  * The HW has no decrement operation, only increment operation.
 939                  * To simulate decrement X from Y using increment operation
 940                  * we need to add UINT32_MAX X times to Y.
 941                  * Each adding of UINT32_MAX decrements Y by 1.
 942                  */
 943                 value *= UINT32_MAX;
 944         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 945         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 946         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 947         item.spec = &tcp;
 948         item.mask = &tcp_mask;
 949         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 950                                              MLX5_MODIFICATION_TYPE_ADD, error);
 951 }
 952
 953 static enum mlx5_modification_field reg_to_field[] = {
 954         [REG_NONE] = MLX5_MODI_OUT_NONE,
 955         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 956         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 957         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 958         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 959         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 960         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 961         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 962         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 963         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 964         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 965 };
 966
 967 /**
 968  * Convert register set to DV specification.
 969  *
 970  * @param[in,out] resource
 971  *   Pointer to the modify-header resource.
 972  * @param[in] action
 973  *   Pointer to action specification.
 974  * @param[out] error
 975  *   Pointer to the error structure.
 976  *
 977  * @return
 978  *   0 on success, a negative errno value otherwise and rte_errno is set.
 979  */
 980 static int
 981 flow_dv_convert_action_set_reg
 982                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 983                          const struct rte_flow_action *action,
 984                          struct rte_flow_error *error)
 985 {
 986         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 987         struct mlx5_modification_cmd *actions = resource->actions;
 988         uint32_t i = resource->actions_num;
 989
 990         if (i >= MLX5_MAX_MODIFY_NUM)
 991                 return rte_flow_error_set(error, EINVAL,
 992                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 993                                           "too many items to modify");
 994         MLX5_ASSERT(conf->id != REG_NONE);
 995         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 996         actions[i] = (struct mlx5_modification_cmd) {
 997                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 998                 .field = reg_to_field[conf->id],
 999         };
1000         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1001         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1002         ++i;
1003         resource->actions_num = i;
1004         return 0;
1005 }
1006
1007 /**
1008  * Convert SET_TAG action to DV specification.
1009  *
1010  * @param[in] dev
1011  *   Pointer to the rte_eth_dev structure.
1012  * @param[in,out] resource
1013  *   Pointer to the modify-header resource.
1014  * @param[in] conf
1015  *   Pointer to action specification.
1016  * @param[out] error
1017  *   Pointer to the error structure.
1018  *
1019  * @return
1020  *   0 on success, a negative errno value otherwise and rte_errno is set.
1021  */
1022 static int
1023 flow_dv_convert_action_set_tag
1024                         (struct rte_eth_dev *dev,
1025                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1026                          const struct rte_flow_action_set_tag *conf,
1027                          struct rte_flow_error *error)
1028 {
1029         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1030         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1031         struct rte_flow_item item = {
1032                 .spec = &data,
1033                 .mask = &mask,
1034         };
1035         struct field_modify_info reg_c_x[] = {
1036                 [1] = {0, 0, 0},
1037         };
1038         enum mlx5_modification_field reg_type;
1039         int ret;
1040
1041         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1042         if (ret < 0)
1043                 return ret;
1044         MLX5_ASSERT(ret != REG_NONE);
1045         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1046         reg_type = reg_to_field[ret];
1047         MLX5_ASSERT(reg_type > 0);
1048         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1049         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1050                                              MLX5_MODIFICATION_TYPE_SET, error);
1051 }
1052
1053 /**
1054  * Convert internal COPY_REG action to DV specification.
1055  *
1056  * @param[in] dev
1057  *   Pointer to the rte_eth_dev structure.
1058  * @param[in,out] res
1059  *   Pointer to the modify-header resource.
1060  * @param[in] action
1061  *   Pointer to action specification.
1062  * @param[out] error
1063  *   Pointer to the error structure.
1064  *
1065  * @return
1066  *   0 on success, a negative errno value otherwise and rte_errno is set.
1067  */
1068 static int
1069 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1070                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1071                                  const struct rte_flow_action *action,
1072                                  struct rte_flow_error *error)
1073 {
1074         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1075         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1076         struct rte_flow_item item = {
1077                 .spec = NULL,
1078                 .mask = &mask,
1079         };
1080         struct field_modify_info reg_src[] = {
1081                 {4, 0, reg_to_field[conf->src]},
1082                 {0, 0, 0},
1083         };
1084         struct field_modify_info reg_dst = {
1085                 .offset = 0,
1086                 .id = reg_to_field[conf->dst],
1087         };
1088         /* Adjust reg_c[0] usage according to reported mask. */
1089         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1090                 struct mlx5_priv *priv = dev->data->dev_private;
1091                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1092
1093                 MLX5_ASSERT(reg_c0);
1094                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1095                 if (conf->dst == REG_C_0) {
1096                         /* Copy to reg_c[0], within mask only. */
1097                         reg_dst.offset = rte_bsf32(reg_c0);
1098                         /*
1099                          * Mask is ignoring the enianness, because
1100                          * there is no conversion in datapath.
1101                          */
1102 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1103                         /* Copy from destination lower bits to reg_c[0]. */
1104                         mask = reg_c0 >> reg_dst.offset;
1105 #else
1106                         /* Copy from destination upper bits to reg_c[0]. */
1107                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1108                                           rte_fls_u32(reg_c0));
1109 #endif
1110                 } else {
1111                         mask = rte_cpu_to_be_32(reg_c0);
1112 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1113                         /* Copy from reg_c[0] to destination lower bits. */
1114                         reg_dst.offset = 0;
1115 #else
1116                         /* Copy from reg_c[0] to destination upper bits. */
1117                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1118                                          (rte_fls_u32(reg_c0) -
1119                                           rte_bsf32(reg_c0));
1120 #endif
1121                 }
1122         }
1123         return flow_dv_convert_modify_action(&item,
1124                                              reg_src, &reg_dst, res,
1125                                              MLX5_MODIFICATION_TYPE_COPY,
1126                                              error);
1127 }
1128
1129 /**
1130  * Convert MARK action to DV specification. This routine is used
1131  * in extensive metadata only and requires metadata register to be
1132  * handled. In legacy mode hardware tag resource is engaged.
1133  *
1134  * @param[in] dev
1135  *   Pointer to the rte_eth_dev structure.
1136  * @param[in] conf
1137  *   Pointer to MARK action specification.
1138  * @param[in,out] resource
1139  *   Pointer to the modify-header resource.
1140  * @param[out] error
1141  *   Pointer to the error structure.
1142  *
1143  * @return
1144  *   0 on success, a negative errno value otherwise and rte_errno is set.
1145  */
1146 static int
1147 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1148                             const struct rte_flow_action_mark *conf,
1149                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1150                             struct rte_flow_error *error)
1151 {
1152         struct mlx5_priv *priv = dev->data->dev_private;
1153         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1154                                            priv->sh->dv_mark_mask);
1155         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1156         struct rte_flow_item item = {
1157                 .spec = &data,
1158                 .mask = &mask,
1159         };
1160         struct field_modify_info reg_c_x[] = {
1161                 {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */
1162                 {0, 0, 0},
1163         };
1164         int reg;
1165
1166         if (!mask)
1167                 return rte_flow_error_set(error, EINVAL,
1168                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1169                                           NULL, "zero mark action mask");
1170         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1171         if (reg < 0)
1172                 return reg;
1173         MLX5_ASSERT(reg > 0);
1174         if (reg == REG_C_0) {
1175                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1176                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1177
1178                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1179                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1180                 mask = rte_cpu_to_be_32(mask << shl_c0);
1181         }
1182         reg_c_x[0].id = reg_to_field[reg];
1183         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1184                                              MLX5_MODIFICATION_TYPE_SET, error);
1185 }
1186
1187 /**
1188  * Get metadata register index for specified steering domain.
1189  *
1190  * @param[in] dev
1191  *   Pointer to the rte_eth_dev structure.
1192  * @param[in] attr
1193  *   Attributes of flow to determine steering domain.
1194  * @param[out] error
1195  *   Pointer to the error structure.
1196  *
1197  * @return
1198  *   positive index on success, a negative errno value otherwise
1199  *   and rte_errno is set.
1200  */
1201 static enum modify_reg
1202 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1203                          const struct rte_flow_attr *attr,
1204                          struct rte_flow_error *error)
1205 {
1206         int reg =
1207                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1208                                           MLX5_METADATA_FDB :
1209                                             attr->egress ?
1210                                             MLX5_METADATA_TX :
1211                                             MLX5_METADATA_RX, 0, error);
1212         if (reg < 0)
1213                 return rte_flow_error_set(error,
1214                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1215                                           NULL, "unavailable "
1216                                           "metadata register");
1217         return reg;
1218 }
1219
1220 /**
1221  * Convert SET_META action to DV specification.
1222  *
1223  * @param[in] dev
1224  *   Pointer to the rte_eth_dev structure.
1225  * @param[in,out] resource
1226  *   Pointer to the modify-header resource.
1227  * @param[in] attr
1228  *   Attributes of flow that includes this item.
1229  * @param[in] conf
1230  *   Pointer to action specification.
1231  * @param[out] error
1232  *   Pointer to the error structure.
1233  *
1234  * @return
1235  *   0 on success, a negative errno value otherwise and rte_errno is set.
1236  */
1237 static int
1238 flow_dv_convert_action_set_meta
1239                         (struct rte_eth_dev *dev,
1240                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1241                          const struct rte_flow_attr *attr,
1242                          const struct rte_flow_action_set_meta *conf,
1243                          struct rte_flow_error *error)
1244 {
1245         uint32_t data = conf->data;
1246         uint32_t mask = conf->mask;
1247         struct rte_flow_item item = {
1248                 .spec = &data,
1249                 .mask = &mask,
1250         };
1251         struct field_modify_info reg_c_x[] = {
1252                 [1] = {0, 0, 0},
1253         };
1254         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1255
1256         if (reg < 0)
1257                 return reg;
1258         /*
1259          * In datapath code there is no endianness
1260          * coversions for perfromance reasons, all
1261          * pattern conversions are done in rte_flow.
1262          */
1263         if (reg == REG_C_0) {
1264                 struct mlx5_priv *priv = dev->data->dev_private;
1265                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1266                 uint32_t shl_c0;
1267
1268                 MLX5_ASSERT(msk_c0);
1269 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1270                 shl_c0 = rte_bsf32(msk_c0);
1271 #else
1272                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1273 #endif
1274                 mask <<= shl_c0;
1275                 data <<= shl_c0;
1276                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1277         }
1278         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1279         /* The routine expects parameters in memory as big-endian ones. */
1280         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1281                                              MLX5_MODIFICATION_TYPE_SET, error);
1282 }
1283
1284 /**
1285  * Convert modify-header set IPv4 DSCP action to DV specification.
1286  *
1287  * @param[in,out] resource
1288  *   Pointer to the modify-header resource.
1289  * @param[in] action
1290  *   Pointer to action specification.
1291  * @param[out] error
1292  *   Pointer to the error structure.
1293  *
1294  * @return
1295  *   0 on success, a negative errno value otherwise and rte_errno is set.
1296  */
1297 static int
1298 flow_dv_convert_action_modify_ipv4_dscp
1299                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1300                          const struct rte_flow_action *action,
1301                          struct rte_flow_error *error)
1302 {
1303         const struct rte_flow_action_set_dscp *conf =
1304                 (const struct rte_flow_action_set_dscp *)(action->conf);
1305         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1306         struct rte_flow_item_ipv4 ipv4;
1307         struct rte_flow_item_ipv4 ipv4_mask;
1308
1309         memset(&ipv4, 0, sizeof(ipv4));
1310         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1311         ipv4.hdr.type_of_service = conf->dscp;
1312         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1313         item.spec = &ipv4;
1314         item.mask = &ipv4_mask;
1315         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1316                                              MLX5_MODIFICATION_TYPE_SET, error);
1317 }
1318
1319 /**
1320  * Convert modify-header set IPv6 DSCP action to DV specification.
1321  *
1322  * @param[in,out] resource
1323  *   Pointer to the modify-header resource.
1324  * @param[in] action
1325  *   Pointer to action specification.
1326  * @param[out] error
1327  *   Pointer to the error structure.
1328  *
1329  * @return
1330  *   0 on success, a negative errno value otherwise and rte_errno is set.
1331  */
1332 static int
1333 flow_dv_convert_action_modify_ipv6_dscp
1334                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1335                          const struct rte_flow_action *action,
1336                          struct rte_flow_error *error)
1337 {
1338         const struct rte_flow_action_set_dscp *conf =
1339                 (const struct rte_flow_action_set_dscp *)(action->conf);
1340         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1341         struct rte_flow_item_ipv6 ipv6;
1342         struct rte_flow_item_ipv6 ipv6_mask;
1343
1344         memset(&ipv6, 0, sizeof(ipv6));
1345         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1346         /*
1347          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1348          * rdma-core only accept the DSCP bits byte aligned start from
1349          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1350          * bits in IPv6 case as rdma-core requires byte aligned value.
1351          */
1352         ipv6.hdr.vtc_flow = conf->dscp;
1353         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1354         item.spec = &ipv6;
1355         item.mask = &ipv6_mask;
1356         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1357                                              MLX5_MODIFICATION_TYPE_SET, error);
1358 }
1359
1360 /**
1361  * Validate MARK item.
1362  *
1363  * @param[in] dev
1364  *   Pointer to the rte_eth_dev structure.
1365  * @param[in] item
1366  *   Item specification.
1367  * @param[in] attr
1368  *   Attributes of flow that includes this item.
1369  * @param[out] error
1370  *   Pointer to error structure.
1371  *
1372  * @return
1373  *   0 on success, a negative errno value otherwise and rte_errno is set.
1374  */
1375 static int
1376 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1377                            const struct rte_flow_item *item,
1378                            const struct rte_flow_attr *attr __rte_unused,
1379                            struct rte_flow_error *error)
1380 {
1381         struct mlx5_priv *priv = dev->data->dev_private;
1382         struct mlx5_dev_config *config = &priv->config;
1383         const struct rte_flow_item_mark *spec = item->spec;
1384         const struct rte_flow_item_mark *mask = item->mask;
1385         const struct rte_flow_item_mark nic_mask = {
1386                 .id = priv->sh->dv_mark_mask,
1387         };
1388         int ret;
1389
1390         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1391                 return rte_flow_error_set(error, ENOTSUP,
1392                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1393                                           "extended metadata feature"
1394                                           " isn't enabled");
1395         if (!mlx5_flow_ext_mreg_supported(dev))
1396                 return rte_flow_error_set(error, ENOTSUP,
1397                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1398                                           "extended metadata register"
1399                                           " isn't supported");
1400         if (!nic_mask.id)
1401                 return rte_flow_error_set(error, ENOTSUP,
1402                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1403                                           "extended metadata register"
1404                                           " isn't available");
1405         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1406         if (ret < 0)
1407                 return ret;
1408         if (!spec)
1409                 return rte_flow_error_set(error, EINVAL,
1410                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1411                                           item->spec,
1412                                           "data cannot be empty");
1413         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1414                 return rte_flow_error_set(error, EINVAL,
1415                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1416                                           &spec->id,
1417                                           "mark id exceeds the limit");
1418         if (!mask)
1419                 mask = &nic_mask;
1420         if (!mask->id)
1421                 return rte_flow_error_set(error, EINVAL,
1422                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1423                                         "mask cannot be zero");
1424
1425         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1426                                         (const uint8_t *)&nic_mask,
1427                                         sizeof(struct rte_flow_item_mark),
1428                                         error);
1429         if (ret < 0)
1430                 return ret;
1431         return 0;
1432 }
1433
1434 /**
1435  * Validate META item.
1436  *
1437  * @param[in] dev
1438  *   Pointer to the rte_eth_dev structure.
1439  * @param[in] item
1440  *   Item specification.
1441  * @param[in] attr
1442  *   Attributes of flow that includes this item.
1443  * @param[out] error
1444  *   Pointer to error structure.
1445  *
1446  * @return
1447  *   0 on success, a negative errno value otherwise and rte_errno is set.
1448  */
1449 static int
1450 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1451                            const struct rte_flow_item *item,
1452                            const struct rte_flow_attr *attr,
1453                            struct rte_flow_error *error)
1454 {
1455         struct mlx5_priv *priv = dev->data->dev_private;
1456         struct mlx5_dev_config *config = &priv->config;
1457         const struct rte_flow_item_meta *spec = item->spec;
1458         const struct rte_flow_item_meta *mask = item->mask;
1459         struct rte_flow_item_meta nic_mask = {
1460                 .data = UINT32_MAX
1461         };
1462         int reg;
1463         int ret;
1464
1465         if (!spec)
1466                 return rte_flow_error_set(error, EINVAL,
1467                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1468                                           item->spec,
1469                                           "data cannot be empty");
1470         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1471                 if (!mlx5_flow_ext_mreg_supported(dev))
1472                         return rte_flow_error_set(error, ENOTSUP,
1473                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1474                                           "extended metadata register"
1475                                           " isn't supported");
1476                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1477                 if (reg < 0)
1478                         return reg;
1479                 if (reg == REG_B)
1480                         return rte_flow_error_set(error, ENOTSUP,
1481                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1482                                           "match on reg_b "
1483                                           "isn't supported");
1484                 if (reg != REG_A)
1485                         nic_mask.data = priv->sh->dv_meta_mask;
1486         }
1487         if (!mask)
1488                 mask = &rte_flow_item_meta_mask;
1489         if (!mask->data)
1490                 return rte_flow_error_set(error, EINVAL,
1491                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1492                                         "mask cannot be zero");
1493
1494         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1495                                         (const uint8_t *)&nic_mask,
1496                                         sizeof(struct rte_flow_item_meta),
1497                                         error);
1498         return ret;
1499 }
1500
1501 /**
1502  * Validate TAG item.
1503  *
1504  * @param[in] dev
1505  *   Pointer to the rte_eth_dev structure.
1506  * @param[in] item
1507  *   Item specification.
1508  * @param[in] attr
1509  *   Attributes of flow that includes this item.
1510  * @param[out] error
1511  *   Pointer to error structure.
1512  *
1513  * @return
1514  *   0 on success, a negative errno value otherwise and rte_errno is set.
1515  */
1516 static int
1517 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1518                           const struct rte_flow_item *item,
1519                           const struct rte_flow_attr *attr __rte_unused,
1520                           struct rte_flow_error *error)
1521 {
1522         const struct rte_flow_item_tag *spec = item->spec;
1523         const struct rte_flow_item_tag *mask = item->mask;
1524         const struct rte_flow_item_tag nic_mask = {
1525                 .data = RTE_BE32(UINT32_MAX),
1526                 .index = 0xff,
1527         };
1528         int ret;
1529
1530         if (!mlx5_flow_ext_mreg_supported(dev))
1531                 return rte_flow_error_set(error, ENOTSUP,
1532                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1533                                           "extensive metadata register"
1534                                           " isn't supported");
1535         if (!spec)
1536                 return rte_flow_error_set(error, EINVAL,
1537                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1538                                           item->spec,
1539                                           "data cannot be empty");
1540         if (!mask)
1541                 mask = &rte_flow_item_tag_mask;
1542         if (!mask->data)
1543                 return rte_flow_error_set(error, EINVAL,
1544                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1545                                         "mask cannot be zero");
1546
1547         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1548                                         (const uint8_t *)&nic_mask,
1549                                         sizeof(struct rte_flow_item_tag),
1550                                         error);
1551         if (ret < 0)
1552                 return ret;
1553         if (mask->index != 0xff)
1554                 return rte_flow_error_set(error, EINVAL,
1555                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1556                                           "partial mask for tag index"
1557                                           " is not supported");
1558         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1559         if (ret < 0)
1560                 return ret;
1561         MLX5_ASSERT(ret != REG_NONE);
1562         return 0;
1563 }
1564
1565 /**
1566  * Validate vport item.
1567  *
1568  * @param[in] dev
1569  *   Pointer to the rte_eth_dev structure.
1570  * @param[in] item
1571  *   Item specification.
1572  * @param[in] attr
1573  *   Attributes of flow that includes this item.
1574  * @param[in] item_flags
1575  *   Bit-fields that holds the items detected until now.
1576  * @param[out] error
1577  *   Pointer to error structure.
1578  *
1579  * @return
1580  *   0 on success, a negative errno value otherwise and rte_errno is set.
1581  */
1582 static int
1583 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1584                               const struct rte_flow_item *item,
1585                               const struct rte_flow_attr *attr,
1586                               uint64_t item_flags,
1587                               struct rte_flow_error *error)
1588 {
1589         const struct rte_flow_item_port_id *spec = item->spec;
1590         const struct rte_flow_item_port_id *mask = item->mask;
1591         const struct rte_flow_item_port_id switch_mask = {
1592                         .id = 0xffffffff,
1593         };
1594         struct mlx5_priv *esw_priv;
1595         struct mlx5_priv *dev_priv;
1596         int ret;
1597
1598         if (!attr->transfer)
1599                 return rte_flow_error_set(error, EINVAL,
1600                                           RTE_FLOW_ERROR_TYPE_ITEM,
1601                                           NULL,
1602                                           "match on port id is valid only"
1603                                           " when transfer flag is enabled");
1604         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1605                 return rte_flow_error_set(error, ENOTSUP,
1606                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1607                                           "multiple source ports are not"
1608                                           " supported");
1609         if (!mask)
1610                 mask = &switch_mask;
1611         if (mask->id != 0xffffffff)
1612                 return rte_flow_error_set(error, ENOTSUP,
1613                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1614                                            mask,
1615                                            "no support for partial mask on"
1616                                            " \"id\" field");
1617         ret = mlx5_flow_item_acceptable
1618                                 (item, (const uint8_t *)mask,
1619                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1620                                  sizeof(struct rte_flow_item_port_id),
1621                                  error);
1622         if (ret)
1623                 return ret;
1624         if (!spec)
1625                 return 0;
1626         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1627         if (!esw_priv)
1628                 return rte_flow_error_set(error, rte_errno,
1629                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1630                                           "failed to obtain E-Switch info for"
1631                                           " port");
1632         dev_priv = mlx5_dev_to_eswitch_info(dev);
1633         if (!dev_priv)
1634                 return rte_flow_error_set(error, rte_errno,
1635                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1636                                           NULL,
1637                                           "failed to obtain E-Switch info");
1638         if (esw_priv->domain_id != dev_priv->domain_id)
1639                 return rte_flow_error_set(error, EINVAL,
1640                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1641                                           "cannot match on a port from a"
1642                                           " different E-Switch");
1643         return 0;
1644 }
1645
1646 /**
1647  * Validate VLAN item.
1648  *
1649  * @param[in] item
1650  *   Item specification.
1651  * @param[in] item_flags
1652  *   Bit-fields that holds the items detected until now.
1653  * @param[in] dev
1654  *   Ethernet device flow is being created on.
1655  * @param[out] error
1656  *   Pointer to error structure.
1657  *
1658  * @return
1659  *   0 on success, a negative errno value otherwise and rte_errno is set.
1660  */
1661 static int
1662 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1663                            uint64_t item_flags,
1664                            struct rte_eth_dev *dev,
1665                            struct rte_flow_error *error)
1666 {
1667         const struct rte_flow_item_vlan *mask = item->mask;
1668         const struct rte_flow_item_vlan nic_mask = {
1669                 .tci = RTE_BE16(UINT16_MAX),
1670                 .inner_type = RTE_BE16(UINT16_MAX),
1671         };
1672         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1673         int ret;
1674         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1675                                         MLX5_FLOW_LAYER_INNER_L4) :
1676                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1677                                         MLX5_FLOW_LAYER_OUTER_L4);
1678         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1679                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1680
1681         if (item_flags & vlanm)
1682                 return rte_flow_error_set(error, EINVAL,
1683                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1684                                           "multiple VLAN layers not supported");
1685         else if ((item_flags & l34m) != 0)
1686                 return rte_flow_error_set(error, EINVAL,
1687                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1688                                           "VLAN cannot follow L3/L4 layer");
1689         if (!mask)
1690                 mask = &rte_flow_item_vlan_mask;
1691         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1692                                         (const uint8_t *)&nic_mask,
1693                                         sizeof(struct rte_flow_item_vlan),
1694                                         error);
1695         if (ret)
1696                 return ret;
1697         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1698                 struct mlx5_priv *priv = dev->data->dev_private;
1699
1700                 if (priv->vmwa_context) {
1701                         /*
1702                          * Non-NULL context means we have a virtual machine
1703                          * and SR-IOV enabled, we have to create VLAN interface
1704                          * to make hypervisor to setup E-Switch vport
1705                          * context correctly. We avoid creating the multiple
1706                          * VLAN interfaces, so we cannot support VLAN tag mask.
1707                          */
1708                         return rte_flow_error_set(error, EINVAL,
1709                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1710                                                   item,
1711                                                   "VLAN tag mask is not"
1712                                                   " supported in virtual"
1713                                                   " environment");
1714                 }
1715         }
1716         return 0;
1717 }
1718
1719 /*
1720  * GTP flags are contained in 1 byte of the format:
1721  * -------------------------------------------
1722  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1723  * |-----------------------------------------|
1724  * | value | Version | PT | Res | E | S | PN |
1725  * -------------------------------------------
1726  *
1727  * Matching is supported only for GTP flags E, S, PN.
1728  */
1729 #define MLX5_GTP_FLAGS_MASK     0x07
1730
1731 /**
1732  * Validate GTP item.
1733  *
1734  * @param[in] dev
1735  *   Pointer to the rte_eth_dev structure.
1736  * @param[in] item
1737  *   Item specification.
1738  * @param[in] item_flags
1739  *   Bit-fields that holds the items detected until now.
1740  * @param[out] error
1741  *   Pointer to error structure.
1742  *
1743  * @return
1744  *   0 on success, a negative errno value otherwise and rte_errno is set.
1745  */
1746 static int
1747 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1748                           const struct rte_flow_item *item,
1749                           uint64_t item_flags,
1750                           struct rte_flow_error *error)
1751 {
1752         struct mlx5_priv *priv = dev->data->dev_private;
1753         const struct rte_flow_item_gtp *spec = item->spec;
1754         const struct rte_flow_item_gtp *mask = item->mask;
1755         const struct rte_flow_item_gtp nic_mask = {
1756                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1757                 .msg_type = 0xff,
1758                 .teid = RTE_BE32(0xffffffff),
1759         };
1760
1761         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1762                 return rte_flow_error_set(error, ENOTSUP,
1763                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1764                                           "GTP support is not enabled");
1765         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1766                 return rte_flow_error_set(error, ENOTSUP,
1767                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1768                                           "multiple tunnel layers not"
1769                                           " supported");
1770         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1771                 return rte_flow_error_set(error, EINVAL,
1772                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1773                                           "no outer UDP layer found");
1774         if (!mask)
1775                 mask = &rte_flow_item_gtp_mask;
1776         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1777                 return rte_flow_error_set(error, ENOTSUP,
1778                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1779                                           "Match is supported for GTP"
1780                                           " flags only");
1781         return mlx5_flow_item_acceptable
1782                 (item, (const uint8_t *)mask,
1783                  (const uint8_t *)&nic_mask,
1784                  sizeof(struct rte_flow_item_gtp),
1785                  error);
1786 }
1787
1788 /**
1789  * Validate the pop VLAN action.
1790  *
1791  * @param[in] dev
1792  *   Pointer to the rte_eth_dev structure.
1793  * @param[in] action_flags
1794  *   Holds the actions detected until now.
1795  * @param[in] action
1796  *   Pointer to the pop vlan action.
1797  * @param[in] item_flags
1798  *   The items found in this flow rule.
1799  * @param[in] attr
1800  *   Pointer to flow attributes.
1801  * @param[out] error
1802  *   Pointer to error structure.
1803  *
1804  * @return
1805  *   0 on success, a negative errno value otherwise and rte_errno is set.
1806  */
1807 static int
1808 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1809                                  uint64_t action_flags,
1810                                  const struct rte_flow_action *action,
1811                                  uint64_t item_flags,
1812                                  const struct rte_flow_attr *attr,
1813                                  struct rte_flow_error *error)
1814 {
1815         const struct mlx5_priv *priv = dev->data->dev_private;
1816
1817         (void)action;
1818         (void)attr;
1819         if (!priv->sh->pop_vlan_action)
1820                 return rte_flow_error_set(error, ENOTSUP,
1821                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1822                                           NULL,
1823                                           "pop vlan action is not supported");
1824         if (attr->egress)
1825                 return rte_flow_error_set(error, ENOTSUP,
1826                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1827                                           NULL,
1828                                           "pop vlan action not supported for "
1829                                           "egress");
1830         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1831                 return rte_flow_error_set(error, ENOTSUP,
1832                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1833                                           "no support for multiple VLAN "
1834                                           "actions");
1835         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1836                 return rte_flow_error_set(error, ENOTSUP,
1837                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1838                                           NULL,
1839                                           "cannot pop vlan without a "
1840                                           "match on (outer) vlan in the flow");
1841         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1842                 return rte_flow_error_set(error, EINVAL,
1843                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1844                                           "wrong action order, port_id should "
1845                                           "be after pop VLAN action");
1846         if (!attr->transfer && priv->representor)
1847                 return rte_flow_error_set(error, ENOTSUP,
1848                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1849                                           "pop vlan action for VF representor "
1850                                           "not supported on NIC table");
1851         return 0;
1852 }
1853
1854 /**
1855  * Get VLAN default info from vlan match info.
1856  *
1857  * @param[in] items
1858  *   the list of item specifications.
1859  * @param[out] vlan
1860  *   pointer VLAN info to fill to.
1861  *
1862  * @return
1863  *   0 on success, a negative errno value otherwise and rte_errno is set.
1864  */
1865 static void
1866 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1867                                   struct rte_vlan_hdr *vlan)
1868 {
1869         const struct rte_flow_item_vlan nic_mask = {
1870                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1871                                 MLX5DV_FLOW_VLAN_VID_MASK),
1872                 .inner_type = RTE_BE16(0xffff),
1873         };
1874
1875         if (items == NULL)
1876                 return;
1877         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1878                 int type = items->type;
1879
1880                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
1881                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
1882                         break;
1883         }
1884         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
1885                 const struct rte_flow_item_vlan *vlan_m = items->mask;
1886                 const struct rte_flow_item_vlan *vlan_v = items->spec;
1887
1888                 /* If VLAN item in pattern doesn't contain data, return here. */
1889                 if (!vlan_v)
1890                         return;
1891                 if (!vlan_m)
1892                         vlan_m = &nic_mask;
1893                 /* Only full match values are accepted */
1894                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1895                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1896                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
1897                         vlan->vlan_tci |=
1898                                 rte_be_to_cpu_16(vlan_v->tci &
1899                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1900                 }
1901                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1902                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1903                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1904                         vlan->vlan_tci |=
1905                                 rte_be_to_cpu_16(vlan_v->tci &
1906                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
1907                 }
1908                 if (vlan_m->inner_type == nic_mask.inner_type)
1909                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1910                                                            vlan_m->inner_type);
1911         }
1912 }
1913
1914 /**
1915  * Validate the push VLAN action.
1916  *
1917  * @param[in] dev
1918  *   Pointer to the rte_eth_dev structure.
1919  * @param[in] action_flags
1920  *   Holds the actions detected until now.
1921  * @param[in] item_flags
1922  *   The items found in this flow rule.
1923  * @param[in] action
1924  *   Pointer to the action structure.
1925  * @param[in] attr
1926  *   Pointer to flow attributes
1927  * @param[out] error
1928  *   Pointer to error structure.
1929  *
1930  * @return
1931  *   0 on success, a negative errno value otherwise and rte_errno is set.
1932  */
1933 static int
1934 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
1935                                   uint64_t action_flags,
1936                                   const struct rte_flow_item_vlan *vlan_m,
1937                                   const struct rte_flow_action *action,
1938                                   const struct rte_flow_attr *attr,
1939                                   struct rte_flow_error *error)
1940 {
1941         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1942         const struct mlx5_priv *priv = dev->data->dev_private;
1943
1944         if (!attr->transfer && attr->ingress)
1945                 return rte_flow_error_set(error, ENOTSUP,
1946                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1947                                           NULL,
1948                                           "push VLAN action not supported for "
1949                                           "ingress");
1950         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1951             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1954                                           "invalid vlan ethertype");
1955         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1956                 return rte_flow_error_set(error, ENOTSUP,
1957                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1958                                           "no support for multiple VLAN "
1959                                           "actions");
1960         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1961                 return rte_flow_error_set(error, EINVAL,
1962                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1963                                           "wrong action order, port_id should "
1964                                           "be after push VLAN");
1965         if (!attr->transfer && priv->representor)
1966                 return rte_flow_error_set(error, ENOTSUP,
1967                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1968                                           "push vlan action for VF representor "
1969                                           "not supported on NIC table");
1970         if (vlan_m &&
1971             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
1972             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
1973                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
1974             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
1975             !(mlx5_flow_find_action
1976                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
1977                 return rte_flow_error_set(error, EINVAL,
1978                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1979                                           "not full match mask on VLAN PCP and "
1980                                           "there is no of_set_vlan_pcp action, "
1981                                           "push VLAN action cannot figure out "
1982                                           "PCP value");
1983         if (vlan_m &&
1984             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
1985             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
1986                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
1987             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
1988             !(mlx5_flow_find_action
1989                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
1990                 return rte_flow_error_set(error, EINVAL,
1991                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1992                                           "not full match mask on VLAN VID and "
1993                                           "there is no of_set_vlan_vid action, "
1994                                           "push VLAN action cannot figure out "
1995                                           "VID value");
1996         (void)attr;
1997         return 0;
1998 }
1999
2000 /**
2001  * Validate the set VLAN PCP.
2002  *
2003  * @param[in] action_flags
2004  *   Holds the actions detected until now.
2005  * @param[in] actions
2006  *   Pointer to the list of actions remaining in the flow rule.
2007  * @param[out] error
2008  *   Pointer to error structure.
2009  *
2010  * @return
2011  *   0 on success, a negative errno value otherwise and rte_errno is set.
2012  */
2013 static int
2014 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2015                                      const struct rte_flow_action actions[],
2016                                      struct rte_flow_error *error)
2017 {
2018         const struct rte_flow_action *action = actions;
2019         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2020
2021         if (conf->vlan_pcp > 7)
2022                 return rte_flow_error_set(error, EINVAL,
2023                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2024                                           "VLAN PCP value is too big");
2025         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2026                 return rte_flow_error_set(error, ENOTSUP,
2027                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2028                                           "set VLAN PCP action must follow "
2029                                           "the push VLAN action");
2030         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2031                 return rte_flow_error_set(error, ENOTSUP,
2032                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2033                                           "Multiple VLAN PCP modification are "
2034                                           "not supported");
2035         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2036                 return rte_flow_error_set(error, EINVAL,
2037                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2038                                           "wrong action order, port_id should "
2039                                           "be after set VLAN PCP");
2040         return 0;
2041 }
2042
2043 /**
2044  * Validate the set VLAN VID.
2045  *
2046  * @param[in] item_flags
2047  *   Holds the items detected in this rule.
2048  * @param[in] action_flags
2049  *   Holds the actions detected until now.
2050  * @param[in] actions
2051  *   Pointer to the list of actions remaining in the flow rule.
2052  * @param[out] error
2053  *   Pointer to error structure.
2054  *
2055  * @return
2056  *   0 on success, a negative errno value otherwise and rte_errno is set.
2057  */
2058 static int
2059 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2060                                      uint64_t action_flags,
2061                                      const struct rte_flow_action actions[],
2062                                      struct rte_flow_error *error)
2063 {
2064         const struct rte_flow_action *action = actions;
2065         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2066
2067         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2068                 return rte_flow_error_set(error, EINVAL,
2069                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2070                                           "VLAN VID value is too big");
2071         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2072             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2073                 return rte_flow_error_set(error, ENOTSUP,
2074                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2075                                           "set VLAN VID action must follow push"
2076                                           " VLAN action or match on VLAN item");
2077         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2078                 return rte_flow_error_set(error, ENOTSUP,
2079                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2080                                           "Multiple VLAN VID modifications are "
2081                                           "not supported");
2082         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2083                 return rte_flow_error_set(error, EINVAL,
2084                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2085                                           "wrong action order, port_id should "
2086                                           "be after set VLAN VID");
2087         return 0;
2088 }
2089
2090 /*
2091  * Validate the FLAG action.
2092  *
2093  * @param[in] dev
2094  *   Pointer to the rte_eth_dev structure.
2095  * @param[in] action_flags
2096  *   Holds the actions detected until now.
2097  * @param[in] attr
2098  *   Pointer to flow attributes
2099  * @param[out] error
2100  *   Pointer to error structure.
2101  *
2102  * @return
2103  *   0 on success, a negative errno value otherwise and rte_errno is set.
2104  */
2105 static int
2106 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2107                              uint64_t action_flags,
2108                              const struct rte_flow_attr *attr,
2109                              struct rte_flow_error *error)
2110 {
2111         struct mlx5_priv *priv = dev->data->dev_private;
2112         struct mlx5_dev_config *config = &priv->config;
2113         int ret;
2114
2115         /* Fall back if no extended metadata register support. */
2116         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2117                 return mlx5_flow_validate_action_flag(action_flags, attr,
2118                                                       error);
2119         /* Extensive metadata mode requires registers. */
2120         if (!mlx5_flow_ext_mreg_supported(dev))
2121                 return rte_flow_error_set(error, ENOTSUP,
2122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2123                                           "no metadata registers "
2124                                           "to support flag action");
2125         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2126                 return rte_flow_error_set(error, ENOTSUP,
2127                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2128                                           "extended metadata register"
2129                                           " isn't available");
2130         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2131         if (ret < 0)
2132                 return ret;
2133         MLX5_ASSERT(ret > 0);
2134         if (action_flags & MLX5_FLOW_ACTION_MARK)
2135                 return rte_flow_error_set(error, EINVAL,
2136                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2137                                           "can't mark and flag in same flow");
2138         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2139                 return rte_flow_error_set(error, EINVAL,
2140                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2141                                           "can't have 2 flag"
2142                                           " actions in same flow");
2143         return 0;
2144 }
2145
2146 /**
2147  * Validate MARK action.
2148  *
2149  * @param[in] dev
2150  *   Pointer to the rte_eth_dev structure.
2151  * @param[in] action
2152  *   Pointer to action.
2153  * @param[in] action_flags
2154  *   Holds the actions detected until now.
2155  * @param[in] attr
2156  *   Pointer to flow attributes
2157  * @param[out] error
2158  *   Pointer to error structure.
2159  *
2160  * @return
2161  *   0 on success, a negative errno value otherwise and rte_errno is set.
2162  */
2163 static int
2164 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2165                              const struct rte_flow_action *action,
2166                              uint64_t action_flags,
2167                              const struct rte_flow_attr *attr,
2168                              struct rte_flow_error *error)
2169 {
2170         struct mlx5_priv *priv = dev->data->dev_private;
2171         struct mlx5_dev_config *config = &priv->config;
2172         const struct rte_flow_action_mark *mark = action->conf;
2173         int ret;
2174
2175         /* Fall back if no extended metadata register support. */
2176         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2177                 return mlx5_flow_validate_action_mark(action, action_flags,
2178                                                       attr, error);
2179         /* Extensive metadata mode requires registers. */
2180         if (!mlx5_flow_ext_mreg_supported(dev))
2181                 return rte_flow_error_set(error, ENOTSUP,
2182                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2183                                           "no metadata registers "
2184                                           "to support mark action");
2185         if (!priv->sh->dv_mark_mask)
2186                 return rte_flow_error_set(error, ENOTSUP,
2187                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2188                                           "extended metadata register"
2189                                           " isn't available");
2190         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2191         if (ret < 0)
2192                 return ret;
2193         MLX5_ASSERT(ret > 0);
2194         if (!mark)
2195                 return rte_flow_error_set(error, EINVAL,
2196                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2197                                           "configuration cannot be null");
2198         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2199                 return rte_flow_error_set(error, EINVAL,
2200                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2201                                           &mark->id,
2202                                           "mark id exceeds the limit");
2203         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2204                 return rte_flow_error_set(error, EINVAL,
2205                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2206                                           "can't flag and mark in same flow");
2207         if (action_flags & MLX5_FLOW_ACTION_MARK)
2208                 return rte_flow_error_set(error, EINVAL,
2209                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2210                                           "can't have 2 mark actions in same"
2211                                           " flow");
2212         return 0;
2213 }
2214
2215 /**
2216  * Validate SET_META action.
2217  *
2218  * @param[in] dev
2219  *   Pointer to the rte_eth_dev structure.
2220  * @param[in] action
2221  *   Pointer to the action structure.
2222  * @param[in] action_flags
2223  *   Holds the actions detected until now.
2224  * @param[in] attr
2225  *   Pointer to flow attributes
2226  * @param[out] error
2227  *   Pointer to error structure.
2228  *
2229  * @return
2230  *   0 on success, a negative errno value otherwise and rte_errno is set.
2231  */
2232 static int
2233 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2234                                  const struct rte_flow_action *action,
2235                                  uint64_t action_flags __rte_unused,
2236                                  const struct rte_flow_attr *attr,
2237                                  struct rte_flow_error *error)
2238 {
2239         const struct rte_flow_action_set_meta *conf;
2240         uint32_t nic_mask = UINT32_MAX;
2241         int reg;
2242
2243         if (!mlx5_flow_ext_mreg_supported(dev))
2244                 return rte_flow_error_set(error, ENOTSUP,
2245                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2246                                           "extended metadata register"
2247                                           " isn't supported");
2248         reg = flow_dv_get_metadata_reg(dev, attr, error);
2249         if (reg < 0)
2250                 return reg;
2251         if (reg != REG_A && reg != REG_B) {
2252                 struct mlx5_priv *priv = dev->data->dev_private;
2253
2254                 nic_mask = priv->sh->dv_meta_mask;
2255         }
2256         if (!(action->conf))
2257                 return rte_flow_error_set(error, EINVAL,
2258                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2259                                           "configuration cannot be null");
2260         conf = (const struct rte_flow_action_set_meta *)action->conf;
2261         if (!conf->mask)
2262                 return rte_flow_error_set(error, EINVAL,
2263                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2264                                           "zero mask doesn't have any effect");
2265         if (conf->mask & ~nic_mask)
2266                 return rte_flow_error_set(error, EINVAL,
2267                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2268                                           "meta data must be within reg C0");
2269         return 0;
2270 }
2271
2272 /**
2273  * Validate SET_TAG action.
2274  *
2275  * @param[in] dev
2276  *   Pointer to the rte_eth_dev structure.
2277  * @param[in] action
2278  *   Pointer to the action structure.
2279  * @param[in] action_flags
2280  *   Holds the actions detected until now.
2281  * @param[in] attr
2282  *   Pointer to flow attributes
2283  * @param[out] error
2284  *   Pointer to error structure.
2285  *
2286  * @return
2287  *   0 on success, a negative errno value otherwise and rte_errno is set.
2288  */
2289 static int
2290 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2291                                 const struct rte_flow_action *action,
2292                                 uint64_t action_flags,
2293                                 const struct rte_flow_attr *attr,
2294                                 struct rte_flow_error *error)
2295 {
2296         const struct rte_flow_action_set_tag *conf;
2297         const uint64_t terminal_action_flags =
2298                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2299                 MLX5_FLOW_ACTION_RSS;
2300         int ret;
2301
2302         if (!mlx5_flow_ext_mreg_supported(dev))
2303                 return rte_flow_error_set(error, ENOTSUP,
2304                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2305                                           "extensive metadata register"
2306                                           " isn't supported");
2307         if (!(action->conf))
2308                 return rte_flow_error_set(error, EINVAL,
2309                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2310                                           "configuration cannot be null");
2311         conf = (const struct rte_flow_action_set_tag *)action->conf;
2312         if (!conf->mask)
2313                 return rte_flow_error_set(error, EINVAL,
2314                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2315                                           "zero mask doesn't have any effect");
2316         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2317         if (ret < 0)
2318                 return ret;
2319         if (!attr->transfer && attr->ingress &&
2320             (action_flags & terminal_action_flags))
2321                 return rte_flow_error_set(error, EINVAL,
2322                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2323                                           "set_tag has no effect"
2324                                           " with terminal actions");
2325         return 0;
2326 }
2327
2328 /**
2329  * Validate count action.
2330  *
2331  * @param[in] dev
2332  *   Pointer to rte_eth_dev structure.
2333  * @param[out] error
2334  *   Pointer to error structure.
2335  *
2336  * @return
2337  *   0 on success, a negative errno value otherwise and rte_errno is set.
2338  */
2339 static int
2340 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2341                               struct rte_flow_error *error)
2342 {
2343         struct mlx5_priv *priv = dev->data->dev_private;
2344
2345         if (!priv->config.devx)
2346                 goto notsup_err;
2347 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2348         return 0;
2349 #endif
2350 notsup_err:
2351         return rte_flow_error_set
2352                       (error, ENOTSUP,
2353                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2354                        NULL,
2355                        "count action not supported");
2356 }
2357
2358 /**
2359  * Validate the L2 encap action.
2360  *
2361  * @param[in] dev
2362  *   Pointer to the rte_eth_dev structure.
2363  * @param[in] action_flags
2364  *   Holds the actions detected until now.
2365  * @param[in] action
2366  *   Pointer to the action structure.
2367  * @param[in] attr
2368  *   Pointer to flow attributes.
2369  * @param[out] error
2370  *   Pointer to error structure.
2371  *
2372  * @return
2373  *   0 on success, a negative errno value otherwise and rte_errno is set.
2374  */
2375 static int
2376 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2377                                  uint64_t action_flags,
2378                                  const struct rte_flow_action *action,
2379                                  const struct rte_flow_attr *attr,
2380                                  struct rte_flow_error *error)
2381 {
2382         const struct mlx5_priv *priv = dev->data->dev_private;
2383
2384         if (!(action->conf))
2385                 return rte_flow_error_set(error, EINVAL,
2386                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2387                                           "configuration cannot be null");
2388         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2389                 return rte_flow_error_set(error, EINVAL,
2390                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2391                                           "can only have a single encap action "
2392                                           "in a flow");
2393         if (!attr->transfer && priv->representor)
2394                 return rte_flow_error_set(error, ENOTSUP,
2395                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2396                                           "encap action for VF representor "
2397                                           "not supported on NIC table");
2398         return 0;
2399 }
2400
2401 /**
2402  * Validate a decap action.
2403  *
2404  * @param[in] dev
2405  *   Pointer to the rte_eth_dev structure.
2406  * @param[in] action_flags
2407  *   Holds the actions detected until now.
2408  * @param[in] attr
2409  *   Pointer to flow attributes
2410  * @param[out] error
2411  *   Pointer to error structure.
2412  *
2413  * @return
2414  *   0 on success, a negative errno value otherwise and rte_errno is set.
2415  */
2416 static int
2417 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2418                               uint64_t action_flags,
2419                               const struct rte_flow_attr *attr,
2420                               struct rte_flow_error *error)
2421 {
2422         const struct mlx5_priv *priv = dev->data->dev_private;
2423
2424         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2425                 return rte_flow_error_set(error, ENOTSUP,
2426                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2427                                           action_flags &
2428                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2429                                           "have a single decap action" : "decap "
2430                                           "after encap is not supported");
2431         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2432                 return rte_flow_error_set(error, EINVAL,
2433                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2434                                           "can't have decap action after"
2435                                           " modify action");
2436         if (attr->egress)
2437                 return rte_flow_error_set(error, ENOTSUP,
2438                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2439                                           NULL,
2440                                           "decap action not supported for "
2441                                           "egress");
2442         if (!attr->transfer && priv->representor)
2443                 return rte_flow_error_set(error, ENOTSUP,
2444                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2445                                           "decap action for VF representor "
2446                                           "not supported on NIC table");
2447         return 0;
2448 }
2449
2450 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2451
2452 /**
2453  * Validate the raw encap and decap actions.
2454  *
2455  * @param[in] dev
2456  *   Pointer to the rte_eth_dev structure.
2457  * @param[in] decap
2458  *   Pointer to the decap action.
2459  * @param[in] encap
2460  *   Pointer to the encap action.
2461  * @param[in] attr
2462  *   Pointer to flow attributes
2463  * @param[in/out] action_flags
2464  *   Holds the actions detected until now.
2465  * @param[out] actions_n
2466  *   pointer to the number of actions counter.
2467  * @param[out] error
2468  *   Pointer to error structure.
2469  *
2470  * @return
2471  *   0 on success, a negative errno value otherwise and rte_errno is set.
2472  */
2473 static int
2474 flow_dv_validate_action_raw_encap_decap
2475         (struct rte_eth_dev *dev,
2476          const struct rte_flow_action_raw_decap *decap,
2477          const struct rte_flow_action_raw_encap *encap,
2478          const struct rte_flow_attr *attr, uint64_t *action_flags,
2479          int *actions_n, struct rte_flow_error *error)
2480 {
2481         const struct mlx5_priv *priv = dev->data->dev_private;
2482         int ret;
2483
2484         if (encap && (!encap->size || !encap->data))
2485                 return rte_flow_error_set(error, EINVAL,
2486                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2487                                           "raw encap data cannot be empty");
2488         if (decap && encap) {
2489                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2490                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2491                         /* L3 encap. */
2492                         decap = NULL;
2493                 else if (encap->size <=
2494                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2495                            decap->size >
2496                            MLX5_ENCAPSULATION_DECISION_SIZE)
2497                         /* L3 decap. */
2498                         encap = NULL;
2499                 else if (encap->size >
2500                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2501                            decap->size >
2502                            MLX5_ENCAPSULATION_DECISION_SIZE)
2503                         /* 2 L2 actions: encap and decap. */
2504                         ;
2505                 else
2506                         return rte_flow_error_set(error,
2507                                 ENOTSUP,
2508                                 RTE_FLOW_ERROR_TYPE_ACTION,
2509                                 NULL, "unsupported too small "
2510                                 "raw decap and too small raw "
2511                                 "encap combination");
2512         }
2513         if (decap) {
2514                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2515                                                     error);
2516                 if (ret < 0)
2517                         return ret;
2518                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2519                 ++(*actions_n);
2520         }
2521         if (encap) {
2522                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2523                         return rte_flow_error_set(error, ENOTSUP,
2524                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2525                                                   NULL,
2526                                                   "small raw encap size");
2527                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2528                         return rte_flow_error_set(error, EINVAL,
2529                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2530                                                   NULL,
2531                                                   "more than one encap action");
2532                 if (!attr->transfer && priv->representor)
2533                         return rte_flow_error_set
2534                                         (error, ENOTSUP,
2535                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2536                                          "encap action for VF representor "
2537                                          "not supported on NIC table");
2538                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2539                 ++(*actions_n);
2540         }
2541         return 0;
2542 }
2543
2544 /**
2545  * Find existing encap/decap resource or create and register a new one.
2546  *
2547  * @param[in, out] dev
2548  *   Pointer to rte_eth_dev structure.
2549  * @param[in, out] resource
2550  *   Pointer to encap/decap resource.
2551  * @parm[in, out] dev_flow
2552  *   Pointer to the dev_flow.
2553  * @param[out] error
2554  *   pointer to error structure.
2555  *
2556  * @return
2557  *   0 on success otherwise -errno and errno is set.
2558  */
2559 static int
2560 flow_dv_encap_decap_resource_register
2561                         (struct rte_eth_dev *dev,
2562                          struct mlx5_flow_dv_encap_decap_resource *resource,
2563                          struct mlx5_flow *dev_flow,
2564                          struct rte_flow_error *error)
2565 {
2566         struct mlx5_priv *priv = dev->data->dev_private;
2567         struct mlx5_dev_ctx_shared *sh = priv->sh;
2568         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2569         struct mlx5dv_dr_domain *domain;
2570         uint32_t idx = 0;
2571         int ret;
2572
2573         resource->flags = dev_flow->dv.group ? 0 : 1;
2574         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2575                 domain = sh->fdb_domain;
2576         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2577                 domain = sh->rx_domain;
2578         else
2579                 domain = sh->tx_domain;
2580         /* Lookup a matching resource from cache. */
2581         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx,
2582                       cache_resource, next) {
2583                 if (resource->reformat_type == cache_resource->reformat_type &&
2584                     resource->ft_type == cache_resource->ft_type &&
2585                     resource->flags == cache_resource->flags &&
2586                     resource->size == cache_resource->size &&
2587                     !memcmp((const void *)resource->buf,
2588                             (const void *)cache_resource->buf,
2589                             resource->size)) {
2590                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2591                                 (void *)cache_resource,
2592                                 rte_atomic32_read(&cache_resource->refcnt));
2593                         rte_atomic32_inc(&cache_resource->refcnt);
2594                         dev_flow->handle->dvh.rix_encap_decap = idx;
2595                         dev_flow->dv.encap_decap = cache_resource;
2596                         return 0;
2597                 }
2598         }
2599         /* Register new encap/decap resource. */
2600         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2601                                        &dev_flow->handle->dvh.rix_encap_decap);
2602         if (!cache_resource)
2603                 return rte_flow_error_set(error, ENOMEM,
2604                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2605                                           "cannot allocate resource memory");
2606         *cache_resource = *resource;
2607         ret = mlx5_flow_os_create_flow_action_packet_reformat
2608                                         (sh->ctx, domain, cache_resource,
2609                                          &cache_resource->action);
2610         if (ret) {
2611                 rte_free(cache_resource);
2612                 return rte_flow_error_set(error, ENOMEM,
2613                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2614                                           NULL, "cannot create action");
2615         }
2616         rte_atomic32_init(&cache_resource->refcnt);
2617         rte_atomic32_inc(&cache_resource->refcnt);
2618         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps,
2619                      dev_flow->handle->dvh.rix_encap_decap, cache_resource,
2620                      next);
2621         dev_flow->dv.encap_decap = cache_resource;
2622         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2623                 (void *)cache_resource,
2624                 rte_atomic32_read(&cache_resource->refcnt));
2625         return 0;
2626 }
2627
2628 /**
2629  * Find existing table jump resource or create and register a new one.
2630  *
2631  * @param[in, out] dev
2632  *   Pointer to rte_eth_dev structure.
2633  * @param[in, out] tbl
2634  *   Pointer to flow table resource.
2635  * @parm[in, out] dev_flow
2636  *   Pointer to the dev_flow.
2637  * @param[out] error
2638  *   pointer to error structure.
2639  *
2640  * @return
2641  *   0 on success otherwise -errno and errno is set.
2642  */
2643 static int
2644 flow_dv_jump_tbl_resource_register
2645                         (struct rte_eth_dev *dev __rte_unused,
2646                          struct mlx5_flow_tbl_resource *tbl,
2647                          struct mlx5_flow *dev_flow,
2648                          struct rte_flow_error *error)
2649 {
2650         struct mlx5_flow_tbl_data_entry *tbl_data =
2651                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2652         int cnt, ret;
2653
2654         MLX5_ASSERT(tbl);
2655         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2656         if (!cnt) {
2657                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2658                                 (tbl->obj, &tbl_data->jump.action);
2659                 if (ret)
2660                         return rte_flow_error_set(error, ENOMEM,
2661                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2662                                         NULL, "cannot create jump action");
2663                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2664                         (void *)&tbl_data->jump, cnt);
2665         } else {
2666                 /* old jump should not make the table ref++. */
2667                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2668                 MLX5_ASSERT(tbl_data->jump.action);
2669                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2670                         (void *)&tbl_data->jump, cnt);
2671         }
2672         rte_atomic32_inc(&tbl_data->jump.refcnt);
2673         dev_flow->handle->rix_jump = tbl_data->idx;
2674         dev_flow->dv.jump = &tbl_data->jump;
2675         return 0;
2676 }
2677
2678 /**
2679  * Find existing default miss resource or create and register a new one.
2680  *
2681  * @param[in, out] dev
2682  *   Pointer to rte_eth_dev structure.
2683  * @param[out] error
2684  *   pointer to error structure.
2685  *
2686  * @return
2687  *   0 on success otherwise -errno and errno is set.
2688  */
2689 static int
2690 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2691                 struct rte_flow_error *error)
2692 {
2693         struct mlx5_priv *priv = dev->data->dev_private;
2694         struct mlx5_dev_ctx_shared *sh = priv->sh;
2695         struct mlx5_flow_default_miss_resource *cache_resource =
2696                         &sh->default_miss;
2697         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2698
2699         if (!cnt) {
2700                 MLX5_ASSERT(cache_resource->action);
2701                 cache_resource->action =
2702                 mlx5_glue->dr_create_flow_action_default_miss();
2703                 if (!cache_resource->action)
2704                         return rte_flow_error_set(error, ENOMEM,
2705                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2706                                         "cannot create default miss action");
2707                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2708                                 (void *)cache_resource->action, cnt);
2709         }
2710         rte_atomic32_inc(&cache_resource->refcnt);
2711         return 0;
2712 }
2713
2714 /**
2715  * Find existing table port ID resource or create and register a new one.
2716  *
2717  * @param[in, out] dev
2718  *   Pointer to rte_eth_dev structure.
2719  * @param[in, out] resource
2720  *   Pointer to port ID action resource.
2721  * @parm[in, out] dev_flow
2722  *   Pointer to the dev_flow.
2723  * @param[out] error
2724  *   pointer to error structure.
2725  *
2726  * @return
2727  *   0 on success otherwise -errno and errno is set.
2728  */
2729 static int
2730 flow_dv_port_id_action_resource_register
2731                         (struct rte_eth_dev *dev,
2732                          struct mlx5_flow_dv_port_id_action_resource *resource,
2733                          struct mlx5_flow *dev_flow,
2734                          struct rte_flow_error *error)
2735 {
2736         struct mlx5_priv *priv = dev->data->dev_private;
2737         struct mlx5_dev_ctx_shared *sh = priv->sh;
2738         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2739         uint32_t idx = 0;
2740         int ret;
2741
2742         /* Lookup a matching resource from cache. */
2743         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
2744                       idx, cache_resource, next) {
2745                 if (resource->port_id == cache_resource->port_id) {
2746                         DRV_LOG(DEBUG, "port id action resource resource %p: "
2747                                 "refcnt %d++",
2748                                 (void *)cache_resource,
2749                                 rte_atomic32_read(&cache_resource->refcnt));
2750                         rte_atomic32_inc(&cache_resource->refcnt);
2751                         dev_flow->handle->rix_port_id_action = idx;
2752                         dev_flow->dv.port_id_action = cache_resource;
2753                         return 0;
2754                 }
2755         }
2756         /* Register new port id action resource. */
2757         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
2758                                        &dev_flow->handle->rix_port_id_action);
2759         if (!cache_resource)
2760                 return rte_flow_error_set(error, ENOMEM,
2761                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2762                                           "cannot allocate resource memory");
2763         *cache_resource = *resource;
2764         ret = mlx5_flow_os_create_flow_action_dest_port
2765                                 (priv->sh->fdb_domain, resource->port_id,
2766                                  &cache_resource->action);
2767         if (ret) {
2768                 rte_free(cache_resource);
2769                 return rte_flow_error_set(error, ENOMEM,
2770                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2771                                           NULL, "cannot create action");
2772         }
2773         rte_atomic32_init(&cache_resource->refcnt);
2774         rte_atomic32_inc(&cache_resource->refcnt);
2775         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
2776                      dev_flow->handle->rix_port_id_action, cache_resource,
2777                      next);
2778         dev_flow->dv.port_id_action = cache_resource;
2779         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2780                 (void *)cache_resource,
2781                 rte_atomic32_read(&cache_resource->refcnt));
2782         return 0;
2783 }
2784
2785 /**
2786  * Find existing push vlan resource or create and register a new one.
2787  *
2788  * @param [in, out] dev
2789  *   Pointer to rte_eth_dev structure.
2790  * @param[in, out] resource
2791  *   Pointer to port ID action resource.
2792  * @parm[in, out] dev_flow
2793  *   Pointer to the dev_flow.
2794  * @param[out] error
2795  *   pointer to error structure.
2796  *
2797  * @return
2798  *   0 on success otherwise -errno and errno is set.
2799  */
2800 static int
2801 flow_dv_push_vlan_action_resource_register
2802                        (struct rte_eth_dev *dev,
2803                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
2804                         struct mlx5_flow *dev_flow,
2805                         struct rte_flow_error *error)
2806 {
2807         struct mlx5_priv *priv = dev->data->dev_private;
2808         struct mlx5_dev_ctx_shared *sh = priv->sh;
2809         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2810         struct mlx5dv_dr_domain *domain;
2811         uint32_t idx = 0;
2812         int ret;
2813
2814         /* Lookup a matching resource from cache. */
2815         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2816                       sh->push_vlan_action_list, idx, cache_resource, next) {
2817                 if (resource->vlan_tag == cache_resource->vlan_tag &&
2818                     resource->ft_type == cache_resource->ft_type) {
2819                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2820                                 "refcnt %d++",
2821                                 (void *)cache_resource,
2822                                 rte_atomic32_read(&cache_resource->refcnt));
2823                         rte_atomic32_inc(&cache_resource->refcnt);
2824                         dev_flow->handle->dvh.rix_push_vlan = idx;
2825                         dev_flow->dv.push_vlan_res = cache_resource;
2826                         return 0;
2827                 }
2828         }
2829         /* Register new push_vlan action resource. */
2830         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2831                                        &dev_flow->handle->dvh.rix_push_vlan);
2832         if (!cache_resource)
2833                 return rte_flow_error_set(error, ENOMEM,
2834                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2835                                           "cannot allocate resource memory");
2836         *cache_resource = *resource;
2837         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2838                 domain = sh->fdb_domain;
2839         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2840                 domain = sh->rx_domain;
2841         else
2842                 domain = sh->tx_domain;
2843         ret = mlx5_flow_os_create_flow_action_push_vlan
2844                                         (domain, resource->vlan_tag,
2845                                          &cache_resource->action);
2846         if (ret) {
2847                 rte_free(cache_resource);
2848                 return rte_flow_error_set(error, ENOMEM,
2849                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2850                                           NULL, "cannot create action");
2851         }
2852         rte_atomic32_init(&cache_resource->refcnt);
2853         rte_atomic32_inc(&cache_resource->refcnt);
2854         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2855                      &sh->push_vlan_action_list,
2856                      dev_flow->handle->dvh.rix_push_vlan,
2857                      cache_resource, next);
2858         dev_flow->dv.push_vlan_res = cache_resource;
2859         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
2860                 (void *)cache_resource,
2861                 rte_atomic32_read(&cache_resource->refcnt));
2862         return 0;
2863 }
2864 /**
2865  * Get the size of specific rte_flow_item_type
2866  *
2867  * @param[in] item_type
2868  *   Tested rte_flow_item_type.
2869  *
2870  * @return
2871  *   sizeof struct item_type, 0 if void or irrelevant.
2872  */
2873 static size_t
2874 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
2875 {
2876         size_t retval;
2877
2878         switch (item_type) {
2879         case RTE_FLOW_ITEM_TYPE_ETH:
2880                 retval = sizeof(struct rte_flow_item_eth);
2881                 break;
2882         case RTE_FLOW_ITEM_TYPE_VLAN:
2883                 retval = sizeof(struct rte_flow_item_vlan);
2884                 break;
2885         case RTE_FLOW_ITEM_TYPE_IPV4:
2886                 retval = sizeof(struct rte_flow_item_ipv4);
2887                 break;
2888         case RTE_FLOW_ITEM_TYPE_IPV6:
2889                 retval = sizeof(struct rte_flow_item_ipv6);
2890                 break;
2891         case RTE_FLOW_ITEM_TYPE_UDP:
2892                 retval = sizeof(struct rte_flow_item_udp);
2893                 break;
2894         case RTE_FLOW_ITEM_TYPE_TCP:
2895                 retval = sizeof(struct rte_flow_item_tcp);
2896                 break;
2897         case RTE_FLOW_ITEM_TYPE_VXLAN:
2898                 retval = sizeof(struct rte_flow_item_vxlan);
2899                 break;
2900         case RTE_FLOW_ITEM_TYPE_GRE:
2901                 retval = sizeof(struct rte_flow_item_gre);
2902                 break;
2903         case RTE_FLOW_ITEM_TYPE_NVGRE:
2904                 retval = sizeof(struct rte_flow_item_nvgre);
2905                 break;
2906         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2907                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
2908                 break;
2909         case RTE_FLOW_ITEM_TYPE_MPLS:
2910                 retval = sizeof(struct rte_flow_item_mpls);
2911                 break;
2912         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
2913         default:
2914                 retval = 0;
2915                 break;
2916         }
2917         return retval;
2918 }
2919
2920 #define MLX5_ENCAP_IPV4_VERSION         0x40
2921 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
2922 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
2923 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
2924 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
2925 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
2926 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
2927
2928 /**
2929  * Convert the encap action data from list of rte_flow_item to raw buffer
2930  *
2931  * @param[in] items
2932  *   Pointer to rte_flow_item objects list.
2933  * @param[out] buf
2934  *   Pointer to the output buffer.
2935  * @param[out] size
2936  *   Pointer to the output buffer size.
2937  * @param[out] error
2938  *   Pointer to the error structure.
2939  *
2940  * @return
2941  *   0 on success, a negative errno value otherwise and rte_errno is set.
2942  */
2943 static int
2944 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
2945                            size_t *size, struct rte_flow_error *error)
2946 {
2947         struct rte_ether_hdr *eth = NULL;
2948         struct rte_vlan_hdr *vlan = NULL;
2949         struct rte_ipv4_hdr *ipv4 = NULL;
2950         struct rte_ipv6_hdr *ipv6 = NULL;
2951         struct rte_udp_hdr *udp = NULL;
2952         struct rte_vxlan_hdr *vxlan = NULL;
2953         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
2954         struct rte_gre_hdr *gre = NULL;
2955         size_t len;
2956         size_t temp_size = 0;
2957
2958         if (!items)
2959                 return rte_flow_error_set(error, EINVAL,
2960                                           RTE_FLOW_ERROR_TYPE_ACTION,
2961                                           NULL, "invalid empty data");
2962         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2963                 len = flow_dv_get_item_len(items->type);
2964                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
2965                         return rte_flow_error_set(error, EINVAL,
2966                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2967                                                   (void *)items->type,
2968                                                   "items total size is too big"
2969                                                   " for encap action");
2970                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
2971                 switch (items->type) {
2972                 case RTE_FLOW_ITEM_TYPE_ETH:
2973                         eth = (struct rte_ether_hdr *)&buf[temp_size];
2974                         break;
2975                 case RTE_FLOW_ITEM_TYPE_VLAN:
2976                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
2977                         if (!eth)
2978                                 return rte_flow_error_set(error, EINVAL,
2979                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2980                                                 (void *)items->type,
2981                                                 "eth header not found");
2982                         if (!eth->ether_type)
2983                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
2984                         break;
2985                 case RTE_FLOW_ITEM_TYPE_IPV4:
2986                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
2987                         if (!vlan && !eth)
2988                                 return rte_flow_error_set(error, EINVAL,
2989                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2990                                                 (void *)items->type,
2991                                                 "neither eth nor vlan"
2992                                                 " header found");
2993                         if (vlan && !vlan->eth_proto)
2994                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
2995                         else if (eth && !eth->ether_type)
2996                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
2997                         if (!ipv4->version_ihl)
2998                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
2999                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3000                         if (!ipv4->time_to_live)
3001                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3002                         break;
3003                 case RTE_FLOW_ITEM_TYPE_IPV6:
3004                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3005                         if (!vlan && !eth)
3006                                 return rte_flow_error_set(error, EINVAL,
3007                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3008                                                 (void *)items->type,
3009                                                 "neither eth nor vlan"
3010                                                 " header found");
3011                         if (vlan && !vlan->eth_proto)
3012                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3013                         else if (eth && !eth->ether_type)
3014                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3015                         if (!ipv6->vtc_flow)
3016                                 ipv6->vtc_flow =
3017                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3018                         if (!ipv6->hop_limits)
3019                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3020                         break;
3021                 case RTE_FLOW_ITEM_TYPE_UDP:
3022                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3023                         if (!ipv4 && !ipv6)
3024                                 return rte_flow_error_set(error, EINVAL,
3025                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3026                                                 (void *)items->type,
3027                                                 "ip header not found");
3028                         if (ipv4 && !ipv4->next_proto_id)
3029                                 ipv4->next_proto_id = IPPROTO_UDP;
3030                         else if (ipv6 && !ipv6->proto)
3031                                 ipv6->proto = IPPROTO_UDP;
3032                         break;
3033                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3034                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3035                         if (!udp)
3036                                 return rte_flow_error_set(error, EINVAL,
3037                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3038                                                 (void *)items->type,
3039                                                 "udp header not found");
3040                         if (!udp->dst_port)
3041                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3042                         if (!vxlan->vx_flags)
3043                                 vxlan->vx_flags =
3044                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3045                         break;
3046                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3047                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3048                         if (!udp)
3049                                 return rte_flow_error_set(error, EINVAL,
3050                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3051                                                 (void *)items->type,
3052                                                 "udp header not found");
3053                         if (!vxlan_gpe->proto)
3054                                 return rte_flow_error_set(error, EINVAL,
3055                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3056                                                 (void *)items->type,
3057                                                 "next protocol not found");
3058                         if (!udp->dst_port)
3059                                 udp->dst_port =
3060                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3061                         if (!vxlan_gpe->vx_flags)
3062                                 vxlan_gpe->vx_flags =
3063                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3064                         break;
3065                 case RTE_FLOW_ITEM_TYPE_GRE:
3066                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3067                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3068                         if (!gre->proto)
3069                                 return rte_flow_error_set(error, EINVAL,
3070                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3071                                                 (void *)items->type,
3072                                                 "next protocol not found");
3073                         if (!ipv4 && !ipv6)
3074                                 return rte_flow_error_set(error, EINVAL,
3075                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3076                                                 (void *)items->type,
3077                                                 "ip header not found");
3078                         if (ipv4 && !ipv4->next_proto_id)
3079                                 ipv4->next_proto_id = IPPROTO_GRE;
3080                         else if (ipv6 && !ipv6->proto)
3081                                 ipv6->proto = IPPROTO_GRE;
3082                         break;
3083                 case RTE_FLOW_ITEM_TYPE_VOID:
3084                         break;
3085                 default:
3086                         return rte_flow_error_set(error, EINVAL,
3087                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3088                                                   (void *)items->type,
3089                                                   "unsupported item type");
3090                         break;
3091                 }
3092                 temp_size += len;
3093         }
3094         *size = temp_size;
3095         return 0;
3096 }
3097
3098 static int
3099 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3100 {
3101         struct rte_ether_hdr *eth = NULL;
3102         struct rte_vlan_hdr *vlan = NULL;
3103         struct rte_ipv6_hdr *ipv6 = NULL;
3104         struct rte_udp_hdr *udp = NULL;
3105         char *next_hdr;
3106         uint16_t proto;
3107
3108         eth = (struct rte_ether_hdr *)data;
3109         next_hdr = (char *)(eth + 1);
3110         proto = RTE_BE16(eth->ether_type);
3111
3112         /* VLAN skipping */
3113         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3114                 vlan = (struct rte_vlan_hdr *)next_hdr;
3115                 proto = RTE_BE16(vlan->eth_proto);
3116                 next_hdr += sizeof(struct rte_vlan_hdr);
3117         }
3118
3119         /* HW calculates IPv4 csum. no need to proceed */
3120         if (proto == RTE_ETHER_TYPE_IPV4)
3121                 return 0;
3122
3123         /* non IPv4/IPv6 header. not supported */
3124         if (proto != RTE_ETHER_TYPE_IPV6) {
3125                 return rte_flow_error_set(error, ENOTSUP,
3126                                           RTE_FLOW_ERROR_TYPE_ACTION,
3127                                           NULL, "Cannot offload non IPv4/IPv6");
3128         }
3129
3130         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3131
3132         /* ignore non UDP */
3133         if (ipv6->proto != IPPROTO_UDP)
3134                 return 0;
3135
3136         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3137         udp->dgram_cksum = 0;
3138
3139         return 0;
3140 }
3141
3142 /**
3143  * Convert L2 encap action to DV specification.
3144  *
3145  * @param[in] dev
3146  *   Pointer to rte_eth_dev structure.
3147  * @param[in] action
3148  *   Pointer to action structure.
3149  * @param[in, out] dev_flow
3150  *   Pointer to the mlx5_flow.
3151  * @param[in] transfer
3152  *   Mark if the flow is E-Switch flow.
3153  * @param[out] error
3154  *   Pointer to the error structure.
3155  *
3156  * @return
3157  *   0 on success, a negative errno value otherwise and rte_errno is set.
3158  */
3159 static int
3160 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3161                                const struct rte_flow_action *action,
3162                                struct mlx5_flow *dev_flow,
3163                                uint8_t transfer,
3164                                struct rte_flow_error *error)
3165 {
3166         const struct rte_flow_item *encap_data;
3167         const struct rte_flow_action_raw_encap *raw_encap_data;
3168         struct mlx5_flow_dv_encap_decap_resource res = {
3169                 .reformat_type =
3170                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3171                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3172                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3173         };
3174
3175         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3176                 raw_encap_data =
3177                         (const struct rte_flow_action_raw_encap *)action->conf;
3178                 res.size = raw_encap_data->size;
3179                 memcpy(res.buf, raw_encap_data->data, res.size);
3180         } else {
3181                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3182                         encap_data =
3183                                 ((const struct rte_flow_action_vxlan_encap *)
3184                                                 action->conf)->definition;
3185                 else
3186                         encap_data =
3187                                 ((const struct rte_flow_action_nvgre_encap *)
3188                                                 action->conf)->definition;
3189                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3190                                                &res.size, error))
3191                         return -rte_errno;
3192         }
3193         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3194                 return -rte_errno;
3195         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3196                 return rte_flow_error_set(error, EINVAL,
3197                                           RTE_FLOW_ERROR_TYPE_ACTION,
3198                                           NULL, "can't create L2 encap action");
3199         return 0;
3200 }
3201
3202 /**
3203  * Convert L2 decap action to DV specification.
3204  *
3205  * @param[in] dev
3206  *   Pointer to rte_eth_dev structure.
3207  * @param[in, out] dev_flow
3208  *   Pointer to the mlx5_flow.
3209  * @param[in] transfer
3210  *   Mark if the flow is E-Switch flow.
3211  * @param[out] error
3212  *   Pointer to the error structure.
3213  *
3214  * @return
3215  *   0 on success, a negative errno value otherwise and rte_errno is set.
3216  */
3217 static int
3218 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3219                                struct mlx5_flow *dev_flow,
3220                                uint8_t transfer,
3221                                struct rte_flow_error *error)
3222 {
3223         struct mlx5_flow_dv_encap_decap_resource res = {
3224                 .size = 0,
3225                 .reformat_type =
3226                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3227                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3228                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3229         };
3230
3231         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3232                 return rte_flow_error_set(error, EINVAL,
3233                                           RTE_FLOW_ERROR_TYPE_ACTION,
3234                                           NULL, "can't create L2 decap action");
3235         return 0;
3236 }
3237
3238 /**
3239  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3240  *
3241  * @param[in] dev
3242  *   Pointer to rte_eth_dev structure.
3243  * @param[in] action
3244  *   Pointer to action structure.
3245  * @param[in, out] dev_flow
3246  *   Pointer to the mlx5_flow.
3247  * @param[in] attr
3248  *   Pointer to the flow attributes.
3249  * @param[out] error
3250  *   Pointer to the error structure.
3251  *
3252  * @return
3253  *   0 on success, a negative errno value otherwise and rte_errno is set.
3254  */
3255 static int
3256 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3257                                 const struct rte_flow_action *action,
3258                                 struct mlx5_flow *dev_flow,
3259                                 const struct rte_flow_attr *attr,
3260                                 struct rte_flow_error *error)
3261 {
3262         const struct rte_flow_action_raw_encap *encap_data;
3263         struct mlx5_flow_dv_encap_decap_resource res;
3264
3265         memset(&res, 0, sizeof(res));
3266         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3267         res.size = encap_data->size;
3268         memcpy(res.buf, encap_data->data, res.size);
3269         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3270                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3271                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3272         if (attr->transfer)
3273                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3274         else
3275                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3276                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3277         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3278                 return rte_flow_error_set(error, EINVAL,
3279                                           RTE_FLOW_ERROR_TYPE_ACTION,
3280                                           NULL, "can't create encap action");
3281         return 0;
3282 }
3283
3284 /**
3285  * Create action push VLAN.
3286  *
3287  * @param[in] dev
3288  *   Pointer to rte_eth_dev structure.
3289  * @param[in] attr
3290  *   Pointer to the flow attributes.
3291  * @param[in] vlan
3292  *   Pointer to the vlan to push to the Ethernet header.
3293  * @param[in, out] dev_flow
3294  *   Pointer to the mlx5_flow.
3295  * @param[out] error
3296  *   Pointer to the error structure.
3297  *
3298  * @return
3299  *   0 on success, a negative errno value otherwise and rte_errno is set.
3300  */
3301 static int
3302 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3303                                 const struct rte_flow_attr *attr,
3304                                 const struct rte_vlan_hdr *vlan,
3305                                 struct mlx5_flow *dev_flow,
3306                                 struct rte_flow_error *error)
3307 {
3308         struct mlx5_flow_dv_push_vlan_action_resource res;
3309
3310         memset(&res, 0, sizeof(res));
3311         res.vlan_tag =
3312                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3313                                  vlan->vlan_tci);
3314         if (attr->transfer)
3315                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3316         else
3317                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3318                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3319         return flow_dv_push_vlan_action_resource_register
3320                                             (dev, &res, dev_flow, error);
3321 }
3322
3323 /**
3324  * Validate the modify-header actions.
3325  *
3326  * @param[in] action_flags
3327  *   Holds the actions detected until now.
3328  * @param[in] action
3329  *   Pointer to the modify action.
3330  * @param[out] error
3331  *   Pointer to error structure.
3332  *
3333  * @return
3334  *   0 on success, a negative errno value otherwise and rte_errno is set.
3335  */
3336 static int
3337 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3338                                    const struct rte_flow_action *action,
3339                                    struct rte_flow_error *error)
3340 {
3341         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3342                 return rte_flow_error_set(error, EINVAL,
3343                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3344                                           NULL, "action configuration not set");
3345         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3346                 return rte_flow_error_set(error, EINVAL,
3347                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3348                                           "can't have encap action before"
3349                                           " modify action");
3350         return 0;
3351 }
3352
3353 /**
3354  * Validate the modify-header MAC address actions.
3355  *
3356  * @param[in] action_flags
3357  *   Holds the actions detected until now.
3358  * @param[in] action
3359  *   Pointer to the modify action.
3360  * @param[in] item_flags
3361  *   Holds the items detected.
3362  * @param[out] error
3363  *   Pointer to error structure.
3364  *
3365  * @return
3366  *   0 on success, a negative errno value otherwise and rte_errno is set.
3367  */
3368 static int
3369 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3370                                    const struct rte_flow_action *action,
3371                                    const uint64_t item_flags,
3372                                    struct rte_flow_error *error)
3373 {
3374         int ret = 0;
3375
3376         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3377         if (!ret) {
3378                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3379                         return rte_flow_error_set(error, EINVAL,
3380                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3381                                                   NULL,
3382                                                   "no L2 item in pattern");
3383         }
3384         return ret;
3385 }
3386
3387 /**
3388  * Validate the modify-header IPv4 address actions.
3389  *
3390  * @param[in] action_flags
3391  *   Holds the actions detected until now.
3392  * @param[in] action
3393  *   Pointer to the modify action.
3394  * @param[in] item_flags
3395  *   Holds the items detected.
3396  * @param[out] error
3397  *   Pointer to error structure.
3398  *
3399  * @return
3400  *   0 on success, a negative errno value otherwise and rte_errno is set.
3401  */
3402 static int
3403 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3404                                     const struct rte_flow_action *action,
3405                                     const uint64_t item_flags,
3406                                     struct rte_flow_error *error)
3407 {
3408         int ret = 0;
3409         uint64_t layer;
3410
3411         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3412         if (!ret) {
3413                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3414                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3415                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3416                 if (!(item_flags & layer))
3417                         return rte_flow_error_set(error, EINVAL,
3418                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3419                                                   NULL,
3420                                                   "no ipv4 item in pattern");
3421         }
3422         return ret;
3423 }
3424
3425 /**
3426  * Validate the modify-header IPv6 address actions.
3427  *
3428  * @param[in] action_flags
3429  *   Holds the actions detected until now.
3430  * @param[in] action
3431  *   Pointer to the modify action.
3432  * @param[in] item_flags
3433  *   Holds the items detected.
3434  * @param[out] error
3435  *   Pointer to error structure.
3436  *
3437  * @return
3438  *   0 on success, a negative errno value otherwise and rte_errno is set.
3439  */
3440 static int
3441 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3442                                     const struct rte_flow_action *action,
3443                                     const uint64_t item_flags,
3444                                     struct rte_flow_error *error)
3445 {
3446         int ret = 0;
3447         uint64_t layer;
3448
3449         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3450         if (!ret) {
3451                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3452                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3453                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3454                 if (!(item_flags & layer))
3455                         return rte_flow_error_set(error, EINVAL,
3456                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3457                                                   NULL,
3458                                                   "no ipv6 item in pattern");
3459         }
3460         return ret;
3461 }
3462
3463 /**
3464  * Validate the modify-header TP actions.
3465  *
3466  * @param[in] action_flags
3467  *   Holds the actions detected until now.
3468  * @param[in] action
3469  *   Pointer to the modify action.
3470  * @param[in] item_flags
3471  *   Holds the items detected.
3472  * @param[out] error
3473  *   Pointer to error structure.
3474  *
3475  * @return
3476  *   0 on success, a negative errno value otherwise and rte_errno is set.
3477  */
3478 static int
3479 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3480                                   const struct rte_flow_action *action,
3481                                   const uint64_t item_flags,
3482                                   struct rte_flow_error *error)
3483 {
3484         int ret = 0;
3485         uint64_t layer;
3486
3487         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3488         if (!ret) {
3489                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3490                                  MLX5_FLOW_LAYER_INNER_L4 :
3491                                  MLX5_FLOW_LAYER_OUTER_L4;
3492                 if (!(item_flags & layer))
3493                         return rte_flow_error_set(error, EINVAL,
3494                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3495                                                   NULL, "no transport layer "
3496                                                   "in pattern");
3497         }
3498         return ret;
3499 }
3500
3501 /**
3502  * Validate the modify-header actions of increment/decrement
3503  * TCP Sequence-number.
3504  *
3505  * @param[in] action_flags
3506  *   Holds the actions detected until now.
3507  * @param[in] action
3508  *   Pointer to the modify action.
3509  * @param[in] item_flags
3510  *   Holds the items detected.
3511  * @param[out] error
3512  *   Pointer to error structure.
3513  *
3514  * @return
3515  *   0 on success, a negative errno value otherwise and rte_errno is set.
3516  */
3517 static int
3518 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3519                                        const struct rte_flow_action *action,
3520                                        const uint64_t item_flags,
3521                                        struct rte_flow_error *error)
3522 {
3523         int ret = 0;
3524         uint64_t layer;
3525
3526         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3527         if (!ret) {
3528                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3529                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3530                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3531                 if (!(item_flags & layer))
3532                         return rte_flow_error_set(error, EINVAL,
3533                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3534                                                   NULL, "no TCP item in"
3535                                                   " pattern");
3536                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3537                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3538                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3539                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3540                         return rte_flow_error_set(error, EINVAL,
3541                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3542                                                   NULL,
3543                                                   "cannot decrease and increase"
3544                                                   " TCP sequence number"
3545                                                   " at the same time");
3546         }
3547         return ret;
3548 }
3549
3550 /**
3551  * Validate the modify-header actions of increment/decrement
3552  * TCP Acknowledgment number.
3553  *
3554  * @param[in] action_flags
3555  *   Holds the actions detected until now.
3556  * @param[in] action
3557  *   Pointer to the modify action.
3558  * @param[in] item_flags
3559  *   Holds the items detected.
3560  * @param[out] error
3561  *   Pointer to error structure.
3562  *
3563  * @return
3564  *   0 on success, a negative errno value otherwise and rte_errno is set.
3565  */
3566 static int
3567 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3568                                        const struct rte_flow_action *action,
3569                                        const uint64_t item_flags,
3570                                        struct rte_flow_error *error)
3571 {
3572         int ret = 0;
3573         uint64_t layer;
3574
3575         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3576         if (!ret) {
3577                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3578                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3579                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3580                 if (!(item_flags & layer))
3581                         return rte_flow_error_set(error, EINVAL,
3582                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3583                                                   NULL, "no TCP item in"
3584                                                   " pattern");
3585                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3586                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3587                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3588                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3589                         return rte_flow_error_set(error, EINVAL,
3590                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3591                                                   NULL,
3592                                                   "cannot decrease and increase"
3593                                                   " TCP acknowledgment number"
3594                                                   " at the same time");
3595         }
3596         return ret;
3597 }
3598
3599 /**
3600  * Validate the modify-header TTL actions.
3601  *
3602  * @param[in] action_flags
3603  *   Holds the actions detected until now.
3604  * @param[in] action
3605  *   Pointer to the modify action.
3606  * @param[in] item_flags
3607  *   Holds the items detected.
3608  * @param[out] error
3609  *   Pointer to error structure.
3610  *
3611  * @return
3612  *   0 on success, a negative errno value otherwise and rte_errno is set.
3613  */
3614 static int
3615 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3616                                    const struct rte_flow_action *action,
3617                                    const uint64_t item_flags,
3618                                    struct rte_flow_error *error)
3619 {
3620         int ret = 0;
3621         uint64_t layer;
3622
3623         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3624         if (!ret) {
3625                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3626                                  MLX5_FLOW_LAYER_INNER_L3 :
3627                                  MLX5_FLOW_LAYER_OUTER_L3;
3628                 if (!(item_flags & layer))
3629                         return rte_flow_error_set(error, EINVAL,
3630                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3631                                                   NULL,
3632                                                   "no IP protocol in pattern");
3633         }
3634         return ret;
3635 }
3636
3637 /**
3638  * Validate jump action.
3639  *
3640  * @param[in] action
3641  *   Pointer to the jump action.
3642  * @param[in] action_flags
3643  *   Holds the actions detected until now.
3644  * @param[in] attributes
3645  *   Pointer to flow attributes
3646  * @param[in] external
3647  *   Action belongs to flow rule created by request external to PMD.
3648  * @param[out] error
3649  *   Pointer to error structure.
3650  *
3651  * @return
3652  *   0 on success, a negative errno value otherwise and rte_errno is set.
3653  */
3654 static int
3655 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3656                              uint64_t action_flags,
3657                              const struct rte_flow_attr *attributes,
3658                              bool external, struct rte_flow_error *error)
3659 {
3660         uint32_t target_group, table;
3661         int ret = 0;
3662
3663         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3664                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3665                 return rte_flow_error_set(error, EINVAL,
3666                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3667                                           "can't have 2 fate actions in"
3668                                           " same flow");
3669         if (action_flags & MLX5_FLOW_ACTION_METER)
3670                 return rte_flow_error_set(error, ENOTSUP,
3671                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3672                                           "jump with meter not support");
3673         if (!action->conf)
3674                 return rte_flow_error_set(error, EINVAL,
3675                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3676                                           NULL, "action configuration not set");
3677         target_group =
3678                 ((const struct rte_flow_action_jump *)action->conf)->group;
3679         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3680                                        true, &table, error);
3681         if (ret)
3682                 return ret;
3683         if (attributes->group == target_group)
3684                 return rte_flow_error_set(error, EINVAL,
3685                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3686                                           "target group must be other than"
3687                                           " the current flow group");
3688         return 0;
3689 }
3690
3691 /*
3692  * Validate the port_id action.
3693  *
3694  * @param[in] dev
3695  *   Pointer to rte_eth_dev structure.
3696  * @param[in] action_flags
3697  *   Bit-fields that holds the actions detected until now.
3698  * @param[in] action
3699  *   Port_id RTE action structure.
3700  * @param[in] attr
3701  *   Attributes of flow that includes this action.
3702  * @param[out] error
3703  *   Pointer to error structure.
3704  *
3705  * @return
3706  *   0 on success, a negative errno value otherwise and rte_errno is set.
3707  */
3708 static int
3709 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3710                                 uint64_t action_flags,
3711                                 const struct rte_flow_action *action,
3712                                 const struct rte_flow_attr *attr,
3713                                 struct rte_flow_error *error)
3714 {
3715         const struct rte_flow_action_port_id *port_id;
3716         struct mlx5_priv *act_priv;
3717         struct mlx5_priv *dev_priv;
3718         uint16_t port;
3719
3720         if (!attr->transfer)
3721                 return rte_flow_error_set(error, ENOTSUP,
3722                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3723                                           NULL,
3724                                           "port id action is valid in transfer"
3725                                           " mode only");
3726         if (!action || !action->conf)
3727                 return rte_flow_error_set(error, ENOTSUP,
3728                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3729                                           NULL,
3730                                           "port id action parameters must be"
3731                                           " specified");
3732         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3733                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3734                 return rte_flow_error_set(error, EINVAL,
3735                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3736                                           "can have only one fate actions in"
3737                                           " a flow");
3738         dev_priv = mlx5_dev_to_eswitch_info(dev);
3739         if (!dev_priv)
3740                 return rte_flow_error_set(error, rte_errno,
3741                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3742                                           NULL,
3743                                           "failed to obtain E-Switch info");
3744         port_id = action->conf;
3745         port = port_id->original ? dev->data->port_id : port_id->id;
3746         act_priv = mlx5_port_to_eswitch_info(port, false);
3747         if (!act_priv)
3748                 return rte_flow_error_set
3749                                 (error, rte_errno,
3750                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3751                                  "failed to obtain E-Switch port id for port");
3752         if (act_priv->domain_id != dev_priv->domain_id)
3753                 return rte_flow_error_set
3754                                 (error, EINVAL,
3755                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3756                                  "port does not belong to"
3757                                  " E-Switch being configured");
3758         return 0;
3759 }
3760
3761 /**
3762  * Get the maximum number of modify header actions.
3763  *
3764  * @param dev
3765  *   Pointer to rte_eth_dev structure.
3766  * @param flags
3767  *   Flags bits to check if root level.
3768  *
3769  * @return
3770  *   Max number of modify header actions device can support.
3771  */
3772 static inline unsigned int
3773 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
3774                               uint64_t flags)
3775 {
3776         /*
3777          * There's no way to directly query the max capacity from FW.
3778          * The maximal value on root table should be assumed to be supported.
3779          */
3780         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
3781                 return MLX5_MAX_MODIFY_NUM;
3782         else
3783                 return MLX5_ROOT_TBL_MODIFY_NUM;
3784 }
3785
3786 /**
3787  * Validate the meter action.
3788  *
3789  * @param[in] dev
3790  *   Pointer to rte_eth_dev structure.
3791  * @param[in] action_flags
3792  *   Bit-fields that holds the actions detected until now.
3793  * @param[in] action
3794  *   Pointer to the meter action.
3795  * @param[in] attr
3796  *   Attributes of flow that includes this action.
3797  * @param[out] error
3798  *   Pointer to error structure.
3799  *
3800  * @return
3801  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3802  */
3803 static int
3804 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3805                                 uint64_t action_flags,
3806                                 const struct rte_flow_action *action,
3807                                 const struct rte_flow_attr *attr,
3808                                 struct rte_flow_error *error)
3809 {
3810         struct mlx5_priv *priv = dev->data->dev_private;
3811         const struct rte_flow_action_meter *am = action->conf;
3812         struct mlx5_flow_meter *fm;
3813
3814         if (!am)
3815                 return rte_flow_error_set(error, EINVAL,
3816                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3817                                           "meter action conf is NULL");
3818
3819         if (action_flags & MLX5_FLOW_ACTION_METER)
3820                 return rte_flow_error_set(error, ENOTSUP,
3821                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3822                                           "meter chaining not support");
3823         if (action_flags & MLX5_FLOW_ACTION_JUMP)
3824                 return rte_flow_error_set(error, ENOTSUP,
3825                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3826                                           "meter with jump not support");
3827         if (!priv->mtr_en)
3828                 return rte_flow_error_set(error, ENOTSUP,
3829                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3830                                           NULL,
3831                                           "meter action not supported");
3832         fm = mlx5_flow_meter_find(priv, am->mtr_id);
3833         if (!fm)
3834                 return rte_flow_error_set(error, EINVAL,
3835                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3836                                           "Meter not found");
3837         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
3838               (!fm->ingress && !attr->ingress && attr->egress) ||
3839               (!fm->egress && !attr->egress && attr->ingress))))
3840                 return rte_flow_error_set(error, EINVAL,
3841                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3842                                           "Flow attributes are either invalid "
3843                                           "or have a conflict with current "
3844                                           "meter attributes");
3845         return 0;
3846 }
3847
3848 /**
3849  * Validate the age action.
3850  *
3851  * @param[in] action_flags
3852  *   Holds the actions detected until now.
3853  * @param[in] action
3854  *   Pointer to the age action.
3855  * @param[in] dev
3856  *   Pointer to the Ethernet device structure.
3857  * @param[out] error
3858  *   Pointer to error structure.
3859  *
3860  * @return
3861  *   0 on success, a negative errno value otherwise and rte_errno is set.
3862  */
3863 static int
3864 flow_dv_validate_action_age(uint64_t action_flags,
3865                             const struct rte_flow_action *action,
3866                             struct rte_eth_dev *dev,
3867                             struct rte_flow_error *error)
3868 {
3869         struct mlx5_priv *priv = dev->data->dev_private;
3870         const struct rte_flow_action_age *age = action->conf;
3871
3872         if (!priv->config.devx || priv->counter_fallback)
3873                 return rte_flow_error_set(error, ENOTSUP,
3874                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3875                                           NULL,
3876                                           "age action not supported");
3877         if (!(action->conf))
3878                 return rte_flow_error_set(error, EINVAL,
3879                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3880                                           "configuration cannot be null");
3881         if (age->timeout >= UINT16_MAX / 2 / 10)
3882                 return rte_flow_error_set(error, ENOTSUP,
3883                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3884                                           "Max age time: 3275 seconds");
3885         if (action_flags & MLX5_FLOW_ACTION_AGE)
3886                 return rte_flow_error_set(error, EINVAL,
3887                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3888                                           "Duplicate age ctions set");
3889         return 0;
3890 }
3891
3892 /**
3893  * Validate the modify-header IPv4 DSCP actions.
3894  *
3895  * @param[in] action_flags
3896  *   Holds the actions detected until now.
3897  * @param[in] action
3898  *   Pointer to the modify action.
3899  * @param[in] item_flags
3900  *   Holds the items detected.
3901  * @param[out] error
3902  *   Pointer to error structure.
3903  *
3904  * @return
3905  *   0 on success, a negative errno value otherwise and rte_errno is set.
3906  */
3907 static int
3908 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
3909                                          const struct rte_flow_action *action,
3910                                          const uint64_t item_flags,
3911                                          struct rte_flow_error *error)
3912 {
3913         int ret = 0;
3914
3915         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3916         if (!ret) {
3917                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
3918                         return rte_flow_error_set(error, EINVAL,
3919                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3920                                                   NULL,
3921                                                   "no ipv4 item in pattern");
3922         }
3923         return ret;
3924 }
3925
3926 /**
3927  * Validate the modify-header IPv6 DSCP actions.
3928  *
3929  * @param[in] action_flags
3930  *   Holds the actions detected until now.
3931  * @param[in] action
3932  *   Pointer to the modify action.
3933  * @param[in] item_flags
3934  *   Holds the items detected.
3935  * @param[out] error
3936  *   Pointer to error structure.
3937  *
3938  * @return
3939  *   0 on success, a negative errno value otherwise and rte_errno is set.
3940  */
3941 static int
3942 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
3943                                          const struct rte_flow_action *action,
3944                                          const uint64_t item_flags,
3945                                          struct rte_flow_error *error)
3946 {
3947         int ret = 0;
3948
3949         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3950         if (!ret) {
3951                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
3952                         return rte_flow_error_set(error, EINVAL,
3953                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3954                                                   NULL,
3955                                                   "no ipv6 item in pattern");
3956         }
3957         return ret;
3958 }
3959
3960 /**
3961  * Find existing modify-header resource or create and register a new one.
3962  *
3963  * @param dev[in, out]
3964  *   Pointer to rte_eth_dev structure.
3965  * @param[in, out] resource
3966  *   Pointer to modify-header resource.
3967  * @parm[in, out] dev_flow
3968  *   Pointer to the dev_flow.
3969  * @param[out] error
3970  *   pointer to error structure.
3971  *
3972  * @return
3973  *   0 on success otherwise -errno and errno is set.
3974  */
3975 static int
3976 flow_dv_modify_hdr_resource_register
3977                         (struct rte_eth_dev *dev,
3978                          struct mlx5_flow_dv_modify_hdr_resource *resource,
3979                          struct mlx5_flow *dev_flow,
3980                          struct rte_flow_error *error)
3981 {
3982         struct mlx5_priv *priv = dev->data->dev_private;
3983         struct mlx5_dev_ctx_shared *sh = priv->sh;
3984         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
3985         struct mlx5dv_dr_domain *ns;
3986         uint32_t actions_len;
3987         int ret;
3988
3989         resource->flags = dev_flow->dv.group ? 0 :
3990                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
3991         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
3992                                     resource->flags))
3993                 return rte_flow_error_set(error, EOVERFLOW,
3994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3995                                           "too many modify header items");
3996         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3997                 ns = sh->fdb_domain;
3998         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
3999                 ns = sh->tx_domain;
4000         else
4001                 ns = sh->rx_domain;
4002         /* Lookup a matching resource from cache. */
4003         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4004         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
4005                 if (resource->ft_type == cache_resource->ft_type &&
4006                     resource->actions_num == cache_resource->actions_num &&
4007                     resource->flags == cache_resource->flags &&
4008                     !memcmp((const void *)resource->actions,
4009                             (const void *)cache_resource->actions,
4010                             actions_len)) {
4011                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4012                                 (void *)cache_resource,
4013                                 rte_atomic32_read(&cache_resource->refcnt));
4014                         rte_atomic32_inc(&cache_resource->refcnt);
4015                         dev_flow->handle->dvh.modify_hdr = cache_resource;
4016                         return 0;
4017                 }
4018         }
4019         /* Register new modify-header resource. */
4020         cache_resource = rte_calloc(__func__, 1,
4021                                     sizeof(*cache_resource) + actions_len, 0);
4022         if (!cache_resource)
4023                 return rte_flow_error_set(error, ENOMEM,
4024                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4025                                           "cannot allocate resource memory");
4026         *cache_resource = *resource;
4027         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4028         ret = mlx5_flow_os_create_flow_action_modify_header
4029                                         (sh->ctx, ns, cache_resource,
4030                                          actions_len, &cache_resource->action);
4031         if (ret) {
4032                 rte_free(cache_resource);
4033                 return rte_flow_error_set(error, ENOMEM,
4034                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4035                                           NULL, "cannot create action");
4036         }
4037         rte_atomic32_init(&cache_resource->refcnt);
4038         rte_atomic32_inc(&cache_resource->refcnt);
4039         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
4040         dev_flow->handle->dvh.modify_hdr = cache_resource;
4041         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4042                 (void *)cache_resource,
4043                 rte_atomic32_read(&cache_resource->refcnt));
4044         return 0;
4045 }
4046
4047 /**
4048  * Get DV flow counter by index.
4049  *
4050  * @param[in] dev
4051  *   Pointer to the Ethernet device structure.
4052  * @param[in] idx
4053  *   mlx5 flow counter index in the container.
4054  * @param[out] ppool
4055  *   mlx5 flow counter pool in the container,
4056  *
4057  * @return
4058  *   Pointer to the counter, NULL otherwise.
4059  */
4060 static struct mlx5_flow_counter *
4061 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4062                            uint32_t idx,
4063                            struct mlx5_flow_counter_pool **ppool)
4064 {
4065         struct mlx5_priv *priv = dev->data->dev_private;
4066         struct mlx5_pools_container *cont;
4067         struct mlx5_flow_counter_pool *pool;
4068         uint32_t batch = 0, age = 0;
4069
4070         idx--;
4071         age = MLX_CNT_IS_AGE(idx);
4072         idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4073         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4074                 idx -= MLX5_CNT_BATCH_OFFSET;
4075                 batch = 1;
4076         }
4077         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4078         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4079         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4080         MLX5_ASSERT(pool);
4081         if (ppool)
4082                 *ppool = pool;
4083         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4084 }
4085
4086 /**
4087  * Check the devx counter belongs to the pool.
4088  *
4089  * @param[in] pool
4090  *   Pointer to the counter pool.
4091  * @param[in] id
4092  *   The counter devx ID.
4093  *
4094  * @return
4095  *   True if counter belongs to the pool, false otherwise.
4096  */
4097 static bool
4098 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4099 {
4100         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4101                    MLX5_COUNTERS_PER_POOL;
4102
4103         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4104                 return true;
4105         return false;
4106 }
4107
4108 /**
4109  * Get a pool by devx counter ID.
4110  *
4111  * @param[in] cont
4112  *   Pointer to the counter container.
4113  * @param[in] id
4114  *   The counter devx ID.
4115  *
4116  * @return
4117  *   The counter pool pointer if exists, NULL otherwise,
4118  */
4119 static struct mlx5_flow_counter_pool *
4120 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4121 {
4122         uint32_t i;
4123
4124         /* Check last used pool. */
4125         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4126             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4127                 return cont->pools[cont->last_pool_idx];
4128         /* ID out of range means no suitable pool in the container. */
4129         if (id > cont->max_id || id < cont->min_id)
4130                 return NULL;
4131         /*
4132          * Find the pool from the end of the container, since mostly counter
4133          * ID is sequence increasing, and the last pool should be the needed
4134          * one.
4135          */
4136         i = rte_atomic16_read(&cont->n_valid);
4137         while (i--) {
4138                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4139
4140                 if (flow_dv_is_counter_in_pool(pool, id))
4141                         return pool;
4142         }
4143         return NULL;
4144 }
4145
4146 /**
4147  * Allocate a new memory for the counter values wrapped by all the needed
4148  * management.
4149  *
4150  * @param[in] dev
4151  *   Pointer to the Ethernet device structure.
4152  * @param[in] raws_n
4153  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4154  *
4155  * @return
4156  *   The new memory management pointer on success, otherwise NULL and rte_errno
4157  *   is set.
4158  */
4159 static struct mlx5_counter_stats_mem_mng *
4160 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4161 {
4162         struct mlx5_priv *priv = dev->data->dev_private;
4163         struct mlx5_dev_ctx_shared *sh = priv->sh;
4164         struct mlx5_devx_mkey_attr mkey_attr;
4165         struct mlx5_counter_stats_mem_mng *mem_mng;
4166         volatile struct flow_counter_stats *raw_data;
4167         int size = (sizeof(struct flow_counter_stats) *
4168                         MLX5_COUNTERS_PER_POOL +
4169                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4170                         sizeof(struct mlx5_counter_stats_mem_mng);
4171         uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
4172         int i;
4173
4174         if (!mem) {
4175                 rte_errno = ENOMEM;
4176                 return NULL;
4177         }
4178         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4179         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4180         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4181                                                  IBV_ACCESS_LOCAL_WRITE);
4182         if (!mem_mng->umem) {
4183                 rte_errno = errno;
4184                 rte_free(mem);
4185                 return NULL;
4186         }
4187         mkey_attr.addr = (uintptr_t)mem;
4188         mkey_attr.size = size;
4189         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4190         mkey_attr.pd = sh->pdn;
4191         mkey_attr.log_entity_size = 0;
4192         mkey_attr.pg_access = 0;
4193         mkey_attr.klm_array = NULL;
4194         mkey_attr.klm_num = 0;
4195         if (priv->config.hca_attr.relaxed_ordering_write &&
4196                 priv->config.hca_attr.relaxed_ordering_read  &&
4197                 !haswell_broadwell_cpu)
4198                 mkey_attr.relaxed_ordering = 1;
4199         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4200         if (!mem_mng->dm) {
4201                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4202                 rte_errno = errno;
4203                 rte_free(mem);
4204                 return NULL;
4205         }
4206         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4207         raw_data = (volatile struct flow_counter_stats *)mem;
4208         for (i = 0; i < raws_n; ++i) {
4209                 mem_mng->raws[i].mem_mng = mem_mng;
4210                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4211         }
4212         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4213         return mem_mng;
4214 }
4215
4216 /**
4217  * Resize a counter container.
4218  *
4219  * @param[in] dev
4220  *   Pointer to the Ethernet device structure.
4221  * @param[in] batch
4222  *   Whether the pool is for counter that was allocated by batch command.
4223  * @param[in] age
4224  *   Whether the pool is for Aging counter.
4225  *
4226  * @return
4227  *   0 on success, otherwise negative errno value and rte_errno is set.
4228  */
4229 static int
4230 flow_dv_container_resize(struct rte_eth_dev *dev,
4231                                 uint32_t batch, uint32_t age)
4232 {
4233         struct mlx5_priv *priv = dev->data->dev_private;
4234         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4235                                                                age);
4236         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4237         void *old_pools = cont->pools;
4238         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4239         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4240         void *pools = rte_calloc(__func__, 1, mem_size, 0);
4241
4242         if (!pools) {
4243                 rte_errno = ENOMEM;
4244                 return -ENOMEM;
4245         }
4246         if (old_pools)
4247                 memcpy(pools, old_pools, cont->n *
4248                                        sizeof(struct mlx5_flow_counter_pool *));
4249         /*
4250          * Fallback mode query the counter directly, no background query
4251          * resources are needed.
4252          */
4253         if (!priv->counter_fallback) {
4254                 int i;
4255
4256                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4257                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4258                 if (!mem_mng) {
4259                         rte_free(pools);
4260                         return -ENOMEM;
4261                 }
4262                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4263                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4264                                          mem_mng->raws +
4265                                          MLX5_CNT_CONTAINER_RESIZE +
4266                                          i, next);
4267         }
4268         rte_spinlock_lock(&cont->resize_sl);
4269         cont->n = resize;
4270         cont->mem_mng = mem_mng;
4271         cont->pools = pools;
4272         rte_spinlock_unlock(&cont->resize_sl);
4273         if (old_pools)
4274                 rte_free(old_pools);
4275         return 0;
4276 }
4277
4278 /**
4279  * Query a devx flow counter.
4280  *
4281  * @param[in] dev
4282  *   Pointer to the Ethernet device structure.
4283  * @param[in] cnt
4284  *   Index to the flow counter.
4285  * @param[out] pkts
4286  *   The statistics value of packets.
4287  * @param[out] bytes
4288  *   The statistics value of bytes.
4289  *
4290  * @return
4291  *   0 on success, otherwise a negative errno value and rte_errno is set.
4292  */
4293 static inline int
4294 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4295                      uint64_t *bytes)
4296 {
4297         struct mlx5_priv *priv = dev->data->dev_private;
4298         struct mlx5_flow_counter_pool *pool = NULL;
4299         struct mlx5_flow_counter *cnt;
4300         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4301         int offset;
4302
4303         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4304         MLX5_ASSERT(pool);
4305         if (counter < MLX5_CNT_BATCH_OFFSET) {
4306                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4307                 if (priv->counter_fallback)
4308                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4309                                         0, pkts, bytes, 0, NULL, NULL, 0);
4310         }
4311
4312         rte_spinlock_lock(&pool->sl);
4313         /*
4314          * The single counters allocation may allocate smaller ID than the
4315          * current allocated in parallel to the host reading.
4316          * In this case the new counter values must be reported as 0.
4317          */
4318         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4319                 *pkts = 0;
4320                 *bytes = 0;
4321         } else {
4322                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4323                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4324                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4325         }
4326         rte_spinlock_unlock(&pool->sl);
4327         return 0;
4328 }
4329
4330 /**
4331  * Create and initialize a new counter pool.
4332  *
4333  * @param[in] dev
4334  *   Pointer to the Ethernet device structure.
4335  * @param[out] dcs
4336  *   The devX counter handle.
4337  * @param[in] batch
4338  *   Whether the pool is for counter that was allocated by batch command.
4339  * @param[in] age
4340  *   Whether the pool is for counter that was allocated for aging.
4341  * @param[in/out] cont_cur
4342  *   Pointer to the container pointer, it will be update in pool resize.
4343  *
4344  * @return
4345  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4346  */
4347 static struct mlx5_flow_counter_pool *
4348 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4349                     uint32_t batch, uint32_t age)
4350 {
4351         struct mlx5_priv *priv = dev->data->dev_private;
4352         struct mlx5_flow_counter_pool *pool;
4353         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4354                                                                age);
4355         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4356         uint32_t size = sizeof(*pool);
4357
4358         if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4359                 return NULL;
4360         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4361         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4362         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4363         pool = rte_calloc(__func__, 1, size, 0);
4364         if (!pool) {
4365                 rte_errno = ENOMEM;
4366                 return NULL;
4367         }
4368         pool->min_dcs = dcs;
4369         if (!priv->counter_fallback)
4370                 pool->raw = cont->mem_mng->raws + n_valid %
4371                                                       MLX5_CNT_CONTAINER_RESIZE;
4372         pool->raw_hw = NULL;
4373         pool->type = 0;
4374         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4375         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4376         pool->query_gen = 0;
4377         rte_spinlock_init(&pool->sl);
4378         TAILQ_INIT(&pool->counters[0]);
4379         TAILQ_INIT(&pool->counters[1]);
4380         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4381         pool->index = n_valid;
4382         cont->pools[n_valid] = pool;
4383         if (!batch) {
4384                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4385
4386                 if (base < cont->min_id)
4387                         cont->min_id = base;
4388                 if (base > cont->max_id)
4389                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4390                 cont->last_pool_idx = pool->index;
4391         }
4392         /* Pool initialization must be updated before host thread access. */
4393         rte_cio_wmb();
4394         rte_atomic16_add(&cont->n_valid, 1);
4395         return pool;
4396 }
4397
4398 /**
4399  * Update the minimum dcs-id for aged or no-aged counter pool.
4400  *
4401  * @param[in] dev
4402  *   Pointer to the Ethernet device structure.
4403  * @param[in] pool
4404  *   Current counter pool.
4405  * @param[in] batch
4406  *   Whether the pool is for counter that was allocated by batch command.
4407  * @param[in] age
4408  *   Whether the counter is for aging.
4409  */
4410 static void
4411 flow_dv_counter_update_min_dcs(struct rte_eth_dev *dev,
4412                         struct mlx5_flow_counter_pool *pool,
4413                         uint32_t batch, uint32_t age)
4414 {
4415         struct mlx5_priv *priv = dev->data->dev_private;
4416         struct mlx5_flow_counter_pool *other;
4417         struct mlx5_pools_container *cont;
4418
4419         cont = MLX5_CNT_CONTAINER(priv->sh, batch, (age ^ 0x1));
4420         other = flow_dv_find_pool_by_id(cont, pool->min_dcs->id);
4421         if (!other)
4422                 return;
4423         if (pool->min_dcs->id < other->min_dcs->id) {
4424                 rte_atomic64_set(&other->a64_dcs,
4425                         rte_atomic64_read(&pool->a64_dcs));
4426         } else {
4427                 rte_atomic64_set(&pool->a64_dcs,
4428                         rte_atomic64_read(&other->a64_dcs));
4429         }
4430 }
4431 /**
4432  * Prepare a new counter and/or a new counter pool.
4433  *
4434  * @param[in] dev
4435  *   Pointer to the Ethernet device structure.
4436  * @param[out] cnt_free
4437  *   Where to put the pointer of a new counter.
4438  * @param[in] batch
4439  *   Whether the pool is for counter that was allocated by batch command.
4440  * @param[in] age
4441  *   Whether the pool is for counter that was allocated for aging.
4442  *
4443  * @return
4444  *   The counter pool pointer and @p cnt_free is set on success,
4445  *   NULL otherwise and rte_errno is set.
4446  */
4447 static struct mlx5_flow_counter_pool *
4448 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4449                              struct mlx5_flow_counter **cnt_free,
4450                              uint32_t batch, uint32_t age)
4451 {
4452         struct mlx5_priv *priv = dev->data->dev_private;
4453         struct mlx5_pools_container *cont;
4454         struct mlx5_flow_counter_pool *pool;
4455         struct mlx5_counters tmp_tq;
4456         struct mlx5_devx_obj *dcs = NULL;
4457         struct mlx5_flow_counter *cnt;
4458         uint32_t i;
4459
4460         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4461         if (!batch) {
4462                 /* bulk_bitmap must be 0 for single counter allocation. */
4463                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4464                 if (!dcs)
4465                         return NULL;
4466                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
4467                 if (!pool) {
4468                         pool = flow_dv_pool_create(dev, dcs, batch, age);
4469                         if (!pool) {
4470                                 mlx5_devx_cmd_destroy(dcs);
4471                                 return NULL;
4472                         }
4473                 } else if (dcs->id < pool->min_dcs->id) {
4474                         rte_atomic64_set(&pool->a64_dcs,
4475                                          (int64_t)(uintptr_t)dcs);
4476                 }
4477                 flow_dv_counter_update_min_dcs(dev,
4478                                                 pool, batch, age);
4479                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4480                 cnt = MLX5_POOL_GET_CNT(pool, i);
4481                 cnt->pool = pool;
4482                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4483                 *cnt_free = cnt;
4484                 return pool;
4485         }
4486         /* bulk_bitmap is in 128 counters units. */
4487         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
4488                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4489         if (!dcs) {
4490                 rte_errno = ENODATA;
4491                 return NULL;
4492         }
4493         pool = flow_dv_pool_create(dev, dcs, batch, age);
4494         if (!pool) {
4495                 mlx5_devx_cmd_destroy(dcs);
4496                 return NULL;
4497         }
4498         TAILQ_INIT(&tmp_tq);
4499         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4500                 cnt = MLX5_POOL_GET_CNT(pool, i);
4501                 cnt->pool = pool;
4502                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4503         }
4504         rte_spinlock_lock(&cont->csl);
4505         TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
4506         rte_spinlock_unlock(&cont->csl);
4507         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4508         (*cnt_free)->pool = pool;
4509         return pool;
4510 }
4511
4512 /**
4513  * Search for existed shared counter.
4514  *
4515  * @param[in] dev
4516  *   Pointer to the Ethernet device structure.
4517  * @param[in] id
4518  *   The shared counter ID to search.
4519  * @param[out] ppool
4520  *   mlx5 flow counter pool in the container,
4521  *
4522  * @return
4523  *   NULL if not existed, otherwise pointer to the shared extend counter.
4524  */
4525 static struct mlx5_flow_counter_ext *
4526 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
4527                               struct mlx5_flow_counter_pool **ppool)
4528 {
4529         struct mlx5_priv *priv = dev->data->dev_private;
4530         union mlx5_l3t_data data;
4531         uint32_t cnt_idx;
4532
4533         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
4534                 return NULL;
4535         cnt_idx = data.dword;
4536         /*
4537          * Shared counters don't have age info. The counter extend is after
4538          * the counter datat structure.
4539          */
4540         return (struct mlx5_flow_counter_ext *)
4541                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
4542 }
4543
4544 /**
4545  * Allocate a flow counter.
4546  *
4547  * @param[in] dev
4548  *   Pointer to the Ethernet device structure.
4549  * @param[in] shared
4550  *   Indicate if this counter is shared with other flows.
4551  * @param[in] id
4552  *   Counter identifier.
4553  * @param[in] group
4554  *   Counter flow group.
4555  * @param[in] age
4556  *   Whether the counter was allocated for aging.
4557  *
4558  * @return
4559  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4560  */
4561 static uint32_t
4562 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
4563                       uint16_t group, uint32_t age)
4564 {
4565         struct mlx5_priv *priv = dev->data->dev_private;
4566         struct mlx5_flow_counter_pool *pool = NULL;
4567         struct mlx5_flow_counter *cnt_free = NULL;
4568         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4569         /*
4570          * Currently group 0 flow counter cannot be assigned to a flow if it is
4571          * not the first one in the batch counter allocation, so it is better
4572          * to allocate counters one by one for these flows in a separate
4573          * container.
4574          * A counter can be shared between different groups so need to take
4575          * shared counters from the single container.
4576          */
4577         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
4578         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4579                                                                age);
4580         uint32_t cnt_idx;
4581
4582         if (!priv->config.devx) {
4583                 rte_errno = ENOTSUP;
4584                 return 0;
4585         }
4586         if (shared) {
4587                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
4588                 if (cnt_ext) {
4589                         if (cnt_ext->ref_cnt + 1 == 0) {
4590                                 rte_errno = E2BIG;
4591                                 return 0;
4592                         }
4593                         cnt_ext->ref_cnt++;
4594                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
4595                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
4596                                   + 1;
4597                         return cnt_idx;
4598                 }
4599         }
4600         /* Get free counters from container. */
4601         rte_spinlock_lock(&cont->csl);
4602         cnt_free = TAILQ_FIRST(&cont->counters);
4603         if (cnt_free)
4604                 TAILQ_REMOVE(&cont->counters, cnt_free, next);
4605         rte_spinlock_unlock(&cont->csl);
4606         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
4607                                                        batch, age))
4608                 goto err;
4609         pool = cnt_free->pool;
4610         if (!batch)
4611                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
4612         /* Create a DV counter action only in the first time usage. */
4613         if (!cnt_free->action) {
4614                 uint16_t offset;
4615                 struct mlx5_devx_obj *dcs;
4616                 int ret;
4617
4618                 if (batch) {
4619                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4620                         dcs = pool->min_dcs;
4621                 } else {
4622                         offset = 0;
4623                         dcs = cnt_ext->dcs;
4624                 }
4625                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
4626                                                             &cnt_free->action);
4627                 if (ret) {
4628                         rte_errno = errno;
4629                         goto err;
4630                 }
4631         }
4632         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4633                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4634         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
4635         cnt_idx += age * MLX5_CNT_AGE_OFFSET;
4636         /* Update the counter reset values. */
4637         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4638                                  &cnt_free->bytes))
4639                 goto err;
4640         if (cnt_ext) {
4641                 cnt_ext->shared = shared;
4642                 cnt_ext->ref_cnt = 1;
4643                 cnt_ext->id = id;
4644                 if (shared) {
4645                         union mlx5_l3t_data data;
4646
4647                         data.dword = cnt_idx;
4648                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
4649                                 return 0;
4650                 }
4651         }
4652         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
4653                 /* Start the asynchronous batch query by the host thread. */
4654                 mlx5_set_query_alarm(priv->sh);
4655         return cnt_idx;
4656 err:
4657         if (cnt_free) {
4658                 cnt_free->pool = pool;
4659                 rte_spinlock_lock(&cont->csl);
4660                 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
4661                 rte_spinlock_unlock(&cont->csl);
4662         }
4663         return 0;
4664 }
4665
4666 /**
4667  * Get age param from counter index.
4668  *
4669  * @param[in] dev
4670  *   Pointer to the Ethernet device structure.
4671  * @param[in] counter
4672  *   Index to the counter handler.
4673  *
4674  * @return
4675  *   The aging parameter specified for the counter index.
4676  */
4677 static struct mlx5_age_param*
4678 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
4679                                 uint32_t counter)
4680 {
4681         struct mlx5_flow_counter *cnt;
4682         struct mlx5_flow_counter_pool *pool = NULL;
4683
4684         flow_dv_counter_get_by_idx(dev, counter, &pool);
4685         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
4686         cnt = MLX5_POOL_GET_CNT(pool, counter);
4687         return MLX5_CNT_TO_AGE(cnt);
4688 }
4689
4690 /**
4691  * Remove a flow counter from aged counter list.
4692  *
4693  * @param[in] dev
4694  *   Pointer to the Ethernet device structure.
4695  * @param[in] counter
4696  *   Index to the counter handler.
4697  * @param[in] cnt
4698  *   Pointer to the counter handler.
4699  */
4700 static void
4701 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
4702                                 uint32_t counter, struct mlx5_flow_counter *cnt)
4703 {
4704         struct mlx5_age_info *age_info;
4705         struct mlx5_age_param *age_param;
4706         struct mlx5_priv *priv = dev->data->dev_private;
4707
4708         age_info = GET_PORT_AGE_INFO(priv);
4709         age_param = flow_dv_counter_idx_get_age(dev, counter);
4710         if (rte_atomic16_cmpset((volatile uint16_t *)
4711                         &age_param->state,
4712                         AGE_CANDIDATE, AGE_FREE)
4713                         != AGE_CANDIDATE) {
4714                 /**
4715                  * We need the lock even it is age timeout,
4716                  * since counter may still in process.
4717                  */
4718                 rte_spinlock_lock(&age_info->aged_sl);
4719                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
4720                 rte_spinlock_unlock(&age_info->aged_sl);
4721         }
4722         rte_atomic16_set(&age_param->state, AGE_FREE);
4723 }
4724 /**
4725  * Release a flow counter.
4726  *
4727  * @param[in] dev
4728  *   Pointer to the Ethernet device structure.
4729  * @param[in] counter
4730  *   Index to the counter handler.
4731  */
4732 static void
4733 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
4734 {
4735         struct mlx5_priv *priv = dev->data->dev_private;
4736         struct mlx5_flow_counter_pool *pool = NULL;
4737         struct mlx5_flow_counter *cnt;
4738         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4739
4740         if (!counter)
4741                 return;
4742         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4743         MLX5_ASSERT(pool);
4744         if (counter < MLX5_CNT_BATCH_OFFSET) {
4745                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4746                 if (cnt_ext) {
4747                         if (--cnt_ext->ref_cnt)
4748                                 return;
4749                         if (cnt_ext->shared)
4750                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
4751                                                      cnt_ext->id);
4752                 }
4753         }
4754         if (IS_AGE_POOL(pool))
4755                 flow_dv_counter_remove_from_age(dev, counter, cnt);
4756         cnt->pool = pool;
4757         /*
4758          * Put the counter back to list to be updated in none fallback mode.
4759          * Currently, we are using two list alternately, while one is in query,
4760          * add the freed counter to the other list based on the pool query_gen
4761          * value. After query finishes, add counter the list to the global
4762          * container counter list. The list changes while query starts. In
4763          * this case, lock will not be needed as query callback and release
4764          * function both operate with the different list.
4765          *
4766          */
4767         if (!priv->counter_fallback)
4768                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
4769         else
4770                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
4771                                   (priv->sh, 0, 0))->counters),
4772                                   cnt, next);
4773 }
4774
4775 /**
4776  * Verify the @p attributes will be correctly understood by the NIC and store
4777  * them in the @p flow if everything is correct.
4778  *
4779  * @param[in] dev
4780  *   Pointer to dev struct.
4781  * @param[in] attributes
4782  *   Pointer to flow attributes
4783  * @param[in] external
4784  *   This flow rule is created by request external to PMD.
4785  * @param[out] error
4786  *   Pointer to error structure.
4787  *
4788  * @return
4789  *   - 0 on success and non root table.
4790  *   - 1 on success and root table.
4791  *   - a negative errno value otherwise and rte_errno is set.
4792  */
4793 static int
4794 flow_dv_validate_attributes(struct rte_eth_dev *dev,
4795                             const struct rte_flow_attr *attributes,
4796                             bool external __rte_unused,
4797                             struct rte_flow_error *error)
4798 {
4799         struct mlx5_priv *priv = dev->data->dev_private;
4800         uint32_t priority_max = priv->config.flow_prio - 1;
4801         int ret = 0;
4802
4803 #ifndef HAVE_MLX5DV_DR
4804         if (attributes->group)
4805                 return rte_flow_error_set(error, ENOTSUP,
4806                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
4807                                           NULL,
4808                                           "groups are not supported");
4809 #else
4810         uint32_t table = 0;
4811
4812         ret = mlx5_flow_group_to_table(attributes, external,
4813                                        attributes->group, !!priv->fdb_def_rule,
4814                                        &table, error);
4815         if (ret)
4816                 return ret;
4817         if (!table)
4818                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4819 #endif
4820         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
4821             attributes->priority >= priority_max)
4822                 return rte_flow_error_set(error, ENOTSUP,
4823                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
4824                                           NULL,
4825                                           "priority out of range");
4826         if (attributes->transfer) {
4827                 if (!priv->config.dv_esw_en)
4828                         return rte_flow_error_set
4829                                 (error, ENOTSUP,
4830                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4831                                  "E-Switch dr is not supported");
4832                 if (!(priv->representor || priv->master))
4833                         return rte_flow_error_set
4834                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4835                                  NULL, "E-Switch configuration can only be"
4836                                  " done by a master or a representor device");
4837                 if (attributes->egress)
4838                         return rte_flow_error_set
4839                                 (error, ENOTSUP,
4840                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
4841                                  "egress is not supported");
4842         }
4843         if (!(attributes->egress ^ attributes->ingress))
4844                 return rte_flow_error_set(error, ENOTSUP,
4845                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
4846                                           "must specify exactly one of "
4847                                           "ingress or egress");
4848         return ret;
4849 }
4850
4851 /**
4852  * Internal validation function. For validating both actions and items.
4853  *
4854  * @param[in] dev
4855  *   Pointer to the rte_eth_dev structure.
4856  * @param[in] attr
4857  *   Pointer to the flow attributes.
4858  * @param[in] items
4859  *   Pointer to the list of items.
4860  * @param[in] actions
4861  *   Pointer to the list of actions.
4862  * @param[in] external
4863  *   This flow rule is created by request external to PMD.
4864  * @param[in] hairpin
4865  *   Number of hairpin TX actions, 0 means classic flow.
4866  * @param[out] error
4867  *   Pointer to the error structure.
4868  *
4869  * @return
4870  *   0 on success, a negative errno value otherwise and rte_errno is set.
4871  */
4872 static int
4873 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
4874                  const struct rte_flow_item items[],
4875                  const struct rte_flow_action actions[],
4876                  bool external, int hairpin, struct rte_flow_error *error)
4877 {
4878         int ret;
4879         uint64_t action_flags = 0;
4880         uint64_t item_flags = 0;
4881         uint64_t last_item = 0;
4882         uint8_t next_protocol = 0xff;
4883         uint16_t ether_type = 0;
4884         int actions_n = 0;
4885         uint8_t item_ipv6_proto = 0;
4886         const struct rte_flow_item *gre_item = NULL;
4887         const struct rte_flow_action_raw_decap *decap;
4888         const struct rte_flow_action_raw_encap *encap;
4889         const struct rte_flow_action_rss *rss;
4890         const struct rte_flow_item_tcp nic_tcp_mask = {
4891                 .hdr = {
4892                         .tcp_flags = 0xFF,
4893                         .src_port = RTE_BE16(UINT16_MAX),
4894                         .dst_port = RTE_BE16(UINT16_MAX),
4895                 }
4896         };
4897         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
4898                 .hdr = {
4899                         .src_addr = RTE_BE32(0xffffffff),
4900                         .dst_addr = RTE_BE32(0xffffffff),
4901                         .type_of_service = 0xff,
4902                         .next_proto_id = 0xff,
4903                         .time_to_live = 0xff,
4904                 },
4905         };
4906         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
4907                 .hdr = {
4908                         .src_addr =
4909                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4910                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4911                         .dst_addr =
4912                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4913                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4914                         .vtc_flow = RTE_BE32(0xffffffff),
4915                         .proto = 0xff,
4916                         .hop_limits = 0xff,
4917                 },
4918         };
4919         struct mlx5_priv *priv = dev->data->dev_private;
4920         struct mlx5_dev_config *dev_conf = &priv->config;
4921         uint16_t queue_index = 0xFFFF;
4922         const struct rte_flow_item_vlan *vlan_m = NULL;
4923         int16_t rw_act_num = 0;
4924         uint64_t is_root;
4925
4926         if (items == NULL)
4927                 return -1;
4928         ret = flow_dv_validate_attributes(dev, attr, external, error);
4929         if (ret < 0)
4930                 return ret;
4931         is_root = (uint64_t)ret;
4932         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4933                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4934                 int type = items->type;
4935
4936                 if (!mlx5_flow_os_item_supported(type))
4937                         return rte_flow_error_set(error, ENOTSUP,
4938                                                   RTE_FLOW_ERROR_TYPE_ITEM,
4939                                                   NULL, "item not supported");
4940                 switch (type) {
4941                 case RTE_FLOW_ITEM_TYPE_VOID:
4942                         break;
4943                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4944                         ret = flow_dv_validate_item_port_id
4945                                         (dev, items, attr, item_flags, error);
4946                         if (ret < 0)
4947                                 return ret;
4948                         last_item = MLX5_FLOW_ITEM_PORT_ID;
4949                         break;
4950                 case RTE_FLOW_ITEM_TYPE_ETH:
4951                         ret = mlx5_flow_validate_item_eth(items, item_flags,
4952                                                           error);
4953                         if (ret < 0)
4954                                 return ret;
4955                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4956                                              MLX5_FLOW_LAYER_OUTER_L2;
4957                         if (items->mask != NULL && items->spec != NULL) {
4958                                 ether_type =
4959                                         ((const struct rte_flow_item_eth *)
4960                                          items->spec)->type;
4961                                 ether_type &=
4962                                         ((const struct rte_flow_item_eth *)
4963                                          items->mask)->type;
4964                                 ether_type = rte_be_to_cpu_16(ether_type);
4965                         } else {
4966                                 ether_type = 0;
4967                         }
4968                         break;
4969                 case RTE_FLOW_ITEM_TYPE_VLAN:
4970                         ret = flow_dv_validate_item_vlan(items, item_flags,
4971                                                          dev, error);
4972                         if (ret < 0)
4973                                 return ret;
4974                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
4975                                              MLX5_FLOW_LAYER_OUTER_VLAN;
4976                         if (items->mask != NULL && items->spec != NULL) {
4977                                 ether_type =
4978                                         ((const struct rte_flow_item_vlan *)
4979                                          items->spec)->inner_type;
4980                                 ether_type &=
4981                                         ((const struct rte_flow_item_vlan *)
4982                                          items->mask)->inner_type;
4983                                 ether_type = rte_be_to_cpu_16(ether_type);
4984                         } else {
4985                                 ether_type = 0;
4986                         }
4987                         /* Store outer VLAN mask for of_push_vlan action. */
4988                         if (!tunnel)
4989                                 vlan_m = items->mask;
4990                         break;
4991                 case RTE_FLOW_ITEM_TYPE_IPV4:
4992                         mlx5_flow_tunnel_ip_check(items, next_protocol,
4993                                                   &item_flags, &tunnel);
4994                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
4995                                                            last_item,
4996                                                            ether_type,
4997                                                            &nic_ipv4_mask,
4998                                                            error);
4999                         if (ret < 0)
5000                                 return ret;
5001                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5002                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5003                         if (items->mask != NULL &&
5004                             ((const struct rte_flow_item_ipv4 *)
5005                              items->mask)->hdr.next_proto_id) {
5006                                 next_protocol =
5007                                         ((const struct rte_flow_item_ipv4 *)
5008                                          (items->spec))->hdr.next_proto_id;
5009                                 next_protocol &=
5010                                         ((const struct rte_flow_item_ipv4 *)
5011                                          (items->mask))->hdr.next_proto_id;
5012                         } else {
5013                                 /* Reset for inner layer. */
5014                                 next_protocol = 0xff;
5015                         }
5016                         break;
5017                 case RTE_FLOW_ITEM_TYPE_IPV6:
5018                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5019                                                   &item_flags, &tunnel);
5020                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5021                                                            last_item,
5022                                                            ether_type,
5023                                                            &nic_ipv6_mask,
5024                                                            error);
5025                         if (ret < 0)
5026                                 return ret;
5027                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5028                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5029                         if (items->mask != NULL &&
5030                             ((const struct rte_flow_item_ipv6 *)
5031                              items->mask)->hdr.proto) {
5032                                 item_ipv6_proto =
5033                                         ((const struct rte_flow_item_ipv6 *)
5034                                          items->spec)->hdr.proto;
5035                                 next_protocol =
5036                                         ((const struct rte_flow_item_ipv6 *)
5037                                          items->spec)->hdr.proto;
5038                                 next_protocol &=
5039                                         ((const struct rte_flow_item_ipv6 *)
5040                                          items->mask)->hdr.proto;
5041                         } else {
5042                                 /* Reset for inner layer. */
5043                                 next_protocol = 0xff;
5044                         }
5045                         break;
5046                 case RTE_FLOW_ITEM_TYPE_TCP:
5047                         ret = mlx5_flow_validate_item_tcp
5048                                                 (items, item_flags,
5049                                                  next_protocol,
5050                                                  &nic_tcp_mask,
5051                                                  error);
5052                         if (ret < 0)
5053                                 return ret;
5054                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5055                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5056                         break;
5057                 case RTE_FLOW_ITEM_TYPE_UDP:
5058                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5059                                                           next_protocol,
5060                                                           error);
5061                         if (ret < 0)
5062                                 return ret;
5063                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5064                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5065                         break;
5066                 case RTE_FLOW_ITEM_TYPE_GRE:
5067                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5068                                                           next_protocol, error);
5069                         if (ret < 0)
5070                                 return ret;
5071                         gre_item = items;
5072                         last_item = MLX5_FLOW_LAYER_GRE;
5073                         break;
5074                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5075                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5076                                                             next_protocol,
5077                                                             error);
5078                         if (ret < 0)
5079                                 return ret;
5080                         last_item = MLX5_FLOW_LAYER_NVGRE;
5081                         break;
5082                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5083                         ret = mlx5_flow_validate_item_gre_key
5084                                 (items, item_flags, gre_item, error);
5085                         if (ret < 0)
5086                                 return ret;
5087                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5088                         break;
5089                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5090                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5091                                                             error);
5092                         if (ret < 0)
5093                                 return ret;
5094                         last_item = MLX5_FLOW_LAYER_VXLAN;
5095                         break;
5096                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5097                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5098                                                                 item_flags, dev,
5099                                                                 error);
5100                         if (ret < 0)
5101                                 return ret;
5102                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5103                         break;
5104                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5105                         ret = mlx5_flow_validate_item_geneve(items,
5106                                                              item_flags, dev,
5107                                                              error);
5108                         if (ret < 0)
5109                                 return ret;
5110                         last_item = MLX5_FLOW_LAYER_GENEVE;
5111                         break;
5112                 case RTE_FLOW_ITEM_TYPE_MPLS:
5113                         ret = mlx5_flow_validate_item_mpls(dev, items,
5114                                                            item_flags,
5115                                                            last_item, error);
5116                         if (ret < 0)
5117                                 return ret;
5118                         last_item = MLX5_FLOW_LAYER_MPLS;
5119                         break;
5120
5121                 case RTE_FLOW_ITEM_TYPE_MARK:
5122                         ret = flow_dv_validate_item_mark(dev, items, attr,
5123                                                          error);
5124                         if (ret < 0)
5125                                 return ret;
5126                         last_item = MLX5_FLOW_ITEM_MARK;
5127                         break;
5128                 case RTE_FLOW_ITEM_TYPE_META:
5129                         ret = flow_dv_validate_item_meta(dev, items, attr,
5130                                                          error);
5131                         if (ret < 0)
5132                                 return ret;
5133                         last_item = MLX5_FLOW_ITEM_METADATA;
5134                         break;
5135                 case RTE_FLOW_ITEM_TYPE_ICMP:
5136                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5137                                                            next_protocol,
5138                                                            error);
5139                         if (ret < 0)
5140                                 return ret;
5141                         last_item = MLX5_FLOW_LAYER_ICMP;
5142                         break;
5143                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5144                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5145                                                             next_protocol,
5146                                                             error);
5147                         if (ret < 0)
5148                                 return ret;
5149                         item_ipv6_proto = IPPROTO_ICMPV6;
5150                         last_item = MLX5_FLOW_LAYER_ICMP6;
5151                         break;
5152                 case RTE_FLOW_ITEM_TYPE_TAG:
5153                         ret = flow_dv_validate_item_tag(dev, items,
5154                                                         attr, error);
5155                         if (ret < 0)
5156                                 return ret;
5157                         last_item = MLX5_FLOW_ITEM_TAG;
5158                         break;
5159                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5160                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5161                         break;
5162                 case RTE_FLOW_ITEM_TYPE_GTP:
5163                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5164                                                         error);
5165                         if (ret < 0)
5166                                 return ret;
5167                         last_item = MLX5_FLOW_LAYER_GTP;
5168                         break;
5169                 default:
5170                         return rte_flow_error_set(error, ENOTSUP,
5171                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5172                                                   NULL, "item not supported");
5173                 }
5174                 item_flags |= last_item;
5175         }
5176         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5177                 int type = actions->type;
5178
5179                 if (!mlx5_flow_os_action_supported(type))
5180                         return rte_flow_error_set(error, ENOTSUP,
5181                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5182                                                   actions,
5183                                                   "action not supported");
5184                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5185                         return rte_flow_error_set(error, ENOTSUP,
5186                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5187                                                   actions, "too many actions");
5188                 switch (type) {
5189                 case RTE_FLOW_ACTION_TYPE_VOID:
5190                         break;
5191                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5192                         ret = flow_dv_validate_action_port_id(dev,
5193                                                               action_flags,
5194                                                               actions,
5195                                                               attr,
5196                                                               error);
5197                         if (ret)
5198                                 return ret;
5199                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5200                         ++actions_n;
5201                         break;
5202                 case RTE_FLOW_ACTION_TYPE_FLAG:
5203                         ret = flow_dv_validate_action_flag(dev, action_flags,
5204                                                            attr, error);
5205                         if (ret < 0)
5206                                 return ret;
5207                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5208                                 /* Count all modify-header actions as one. */
5209                                 if (!(action_flags &
5210                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5211                                         ++actions_n;
5212                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5213                                                 MLX5_FLOW_ACTION_MARK_EXT;
5214                         } else {
5215                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5216                                 ++actions_n;
5217                         }
5218                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5219                         break;
5220                 case RTE_FLOW_ACTION_TYPE_MARK:
5221                         ret = flow_dv_validate_action_mark(dev, actions,
5222                                                            action_flags,
5223                                                            attr, error);
5224                         if (ret < 0)
5225                                 return ret;
5226                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5227                                 /* Count all modify-header actions as one. */
5228                                 if (!(action_flags &
5229                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5230                                         ++actions_n;
5231                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5232                                                 MLX5_FLOW_ACTION_MARK_EXT;
5233                         } else {
5234                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5235                                 ++actions_n;
5236                         }
5237                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5238                         break;
5239                 case RTE_FLOW_ACTION_TYPE_SET_META:
5240                         ret = flow_dv_validate_action_set_meta(dev, actions,
5241                                                                action_flags,
5242                                                                attr, error);
5243                         if (ret < 0)
5244                                 return ret;
5245                         /* Count all modify-header actions as one action. */
5246                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5247                                 ++actions_n;
5248                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5249                         rw_act_num += MLX5_ACT_NUM_SET_META;
5250                         break;
5251                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5252                         ret = flow_dv_validate_action_set_tag(dev, actions,
5253                                                               action_flags,
5254                                                               attr, error);
5255                         if (ret < 0)
5256                                 return ret;
5257                         /* Count all modify-header actions as one action. */
5258                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5259                                 ++actions_n;
5260                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5261                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5262                         break;
5263                 case RTE_FLOW_ACTION_TYPE_DROP:
5264                         ret = mlx5_flow_validate_action_drop(action_flags,
5265                                                              attr, error);
5266                         if (ret < 0)
5267                                 return ret;
5268                         action_flags |= MLX5_FLOW_ACTION_DROP;
5269                         ++actions_n;
5270                         break;
5271                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5272                         ret = mlx5_flow_validate_action_queue(actions,
5273                                                               action_flags, dev,
5274                                                               attr, error);
5275                         if (ret < 0)
5276                                 return ret;
5277                         queue_index = ((const struct rte_flow_action_queue *)
5278                                                         (actions->conf))->index;
5279                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5280                         ++actions_n;
5281                         break;
5282                 case RTE_FLOW_ACTION_TYPE_RSS:
5283                         rss = actions->conf;
5284                         ret = mlx5_flow_validate_action_rss(actions,
5285                                                             action_flags, dev,
5286                                                             attr, item_flags,
5287                                                             error);
5288                         if (ret < 0)
5289                                 return ret;
5290                         if (rss != NULL && rss->queue_num)
5291                                 queue_index = rss->queue[0];
5292                         action_flags |= MLX5_FLOW_ACTION_RSS;
5293                         ++actions_n;
5294                         break;
5295                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5296                         ret =
5297                         mlx5_flow_validate_action_default_miss(action_flags,
5298                                         attr, error);
5299                         if (ret < 0)
5300                                 return ret;
5301                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5302                         ++actions_n;
5303                         break;
5304                 case RTE_FLOW_ACTION_TYPE_COUNT:
5305                         ret = flow_dv_validate_action_count(dev, error);
5306                         if (ret < 0)
5307                                 return ret;
5308                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5309                         ++actions_n;
5310                         break;
5311                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5312                         if (flow_dv_validate_action_pop_vlan(dev,
5313                                                              action_flags,
5314                                                              actions,
5315                                                              item_flags, attr,
5316                                                              error))
5317                                 return -rte_errno;
5318                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5319                         ++actions_n;
5320                         break;
5321                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5322                         ret = flow_dv_validate_action_push_vlan(dev,
5323                                                                 action_flags,
5324                                                                 vlan_m,
5325                                                                 actions, attr,
5326                                                                 error);
5327                         if (ret < 0)
5328                                 return ret;
5329                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5330                         ++actions_n;
5331                         break;
5332                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5333                         ret = flow_dv_validate_action_set_vlan_pcp
5334                                                 (action_flags, actions, error);
5335                         if (ret < 0)
5336                                 return ret;
5337                         /* Count PCP with push_vlan command. */
5338                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5339                         break;
5340                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5341                         ret = flow_dv_validate_action_set_vlan_vid
5342                                                 (item_flags, action_flags,
5343                                                  actions, error);
5344                         if (ret < 0)
5345                                 return ret;
5346                         /* Count VID with push_vlan command. */
5347                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5348                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5349                         break;
5350                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5351                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5352                         ret = flow_dv_validate_action_l2_encap(dev,
5353                                                                action_flags,
5354                                                                actions, attr,
5355                                                                error);
5356                         if (ret < 0)
5357                                 return ret;
5358                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5359                         ++actions_n;
5360                         break;
5361                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5362                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5363                         ret = flow_dv_validate_action_decap(dev, action_flags,
5364                                                             attr, error);
5365                         if (ret < 0)
5366                                 return ret;
5367                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5368                         ++actions_n;
5369                         break;
5370                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5371                         ret = flow_dv_validate_action_raw_encap_decap
5372                                 (dev, NULL, actions->conf, attr, &action_flags,
5373                                  &actions_n, error);
5374                         if (ret < 0)
5375                                 return ret;
5376                         break;
5377                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5378                         decap = actions->conf;
5379                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5380                                 ;
5381                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5382                                 encap = NULL;
5383                                 actions--;
5384                         } else {
5385                                 encap = actions->conf;
5386                         }
5387                         ret = flow_dv_validate_action_raw_encap_decap
5388                                            (dev,
5389                                             decap ? decap : &empty_decap, encap,
5390                                             attr, &action_flags, &actions_n,
5391                                             error);
5392                         if (ret < 0)
5393                                 return ret;
5394                         break;
5395                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5396                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5397                         ret = flow_dv_validate_action_modify_mac(action_flags,
5398                                                                  actions,
5399                                                                  item_flags,
5400                                                                  error);
5401                         if (ret < 0)
5402                                 return ret;
5403                         /* Count all modify-header actions as one action. */
5404                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5405                                 ++actions_n;
5406                         action_flags |= actions->type ==
5407                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5408                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5409                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5410                         /*
5411                          * Even if the source and destination MAC addresses have
5412                          * overlap in the header with 4B alignment, the convert
5413                          * function will handle them separately and 4 SW actions
5414                          * will be created. And 2 actions will be added each
5415                          * time no matter how many bytes of address will be set.
5416                          */
5417                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5418                         break;
5419                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5420                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5421                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5422                                                                   actions,
5423                                                                   item_flags,
5424                                                                   error);
5425                         if (ret < 0)
5426                                 return ret;
5427                         /* Count all modify-header actions as one action. */
5428                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5429                                 ++actions_n;
5430                         action_flags |= actions->type ==
5431                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5432                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5433                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5434                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5435                         break;
5436                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5437                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5438                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5439                                                                   actions,
5440                                                                   item_flags,
5441                                                                   error);
5442                         if (ret < 0)
5443                                 return ret;
5444                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5445                                 return rte_flow_error_set(error, ENOTSUP,
5446                                         RTE_FLOW_ERROR_TYPE_ACTION,
5447                                         actions,
5448                                         "Can't change header "
5449                                         "with ICMPv6 proto");
5450                         /* Count all modify-header actions as one action. */
5451                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5452                                 ++actions_n;
5453                         action_flags |= actions->type ==
5454                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5455                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5456                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5457                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5458                         break;
5459                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5460                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5461                         ret = flow_dv_validate_action_modify_tp(action_flags,
5462                                                                 actions,
5463                                                                 item_flags,
5464                                                                 error);
5465                         if (ret < 0)
5466                                 return ret;
5467                         /* Count all modify-header actions as one action. */
5468                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5469                                 ++actions_n;
5470                         action_flags |= actions->type ==
5471                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5472                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5473                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5474                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5475                         break;
5476                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5477                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5478                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5479                                                                  actions,
5480                                                                  item_flags,
5481                                                                  error);
5482                         if (ret < 0)
5483                                 return ret;
5484                         /* Count all modify-header actions as one action. */
5485                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5486                                 ++actions_n;
5487                         action_flags |= actions->type ==
5488                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5489                                                 MLX5_FLOW_ACTION_SET_TTL :
5490                                                 MLX5_FLOW_ACTION_DEC_TTL;
5491                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5492                         break;
5493                 case RTE_FLOW_ACTION_TYPE_JUMP:
5494                         ret = flow_dv_validate_action_jump(actions,
5495                                                            action_flags,
5496                                                            attr, external,
5497                                                            error);
5498                         if (ret)
5499                                 return ret;
5500                         ++actions_n;
5501                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5502                         break;
5503                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5504                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5505                         ret = flow_dv_validate_action_modify_tcp_seq
5506                                                                 (action_flags,
5507                                                                  actions,
5508                                                                  item_flags,
5509                                                                  error);
5510                         if (ret < 0)
5511                                 return ret;
5512                         /* Count all modify-header actions as one action. */
5513                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5514                                 ++actions_n;
5515                         action_flags |= actions->type ==
5516                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5517                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5518                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5519                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5520                         break;
5521                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5522                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5523                         ret = flow_dv_validate_action_modify_tcp_ack
5524                                                                 (action_flags,
5525                                                                  actions,
5526                                                                  item_flags,
5527                                                                  error);
5528                         if (ret < 0)
5529                                 return ret;
5530                         /* Count all modify-header actions as one action. */
5531                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5532                                 ++actions_n;
5533                         action_flags |= actions->type ==
5534                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5535                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5536                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5537                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5538                         break;
5539                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5540                         break;
5541                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5542                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5543                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5544                         break;
5545                 case RTE_FLOW_ACTION_TYPE_METER:
5546                         ret = mlx5_flow_validate_action_meter(dev,
5547                                                               action_flags,
5548                                                               actions, attr,
5549                                                               error);
5550                         if (ret < 0)
5551                                 return ret;
5552                         action_flags |= MLX5_FLOW_ACTION_METER;
5553                         ++actions_n;
5554                         /* Meter action will add one more TAG action. */
5555                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5556                         break;
5557                 case RTE_FLOW_ACTION_TYPE_AGE:
5558                         ret = flow_dv_validate_action_age(action_flags,
5559                                                           actions, dev,
5560                                                           error);
5561                         if (ret < 0)
5562                                 return ret;
5563                         action_flags |= MLX5_FLOW_ACTION_AGE;
5564                         ++actions_n;
5565                         break;
5566                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5567                         ret = flow_dv_validate_action_modify_ipv4_dscp
5568                                                          (action_flags,
5569                                                           actions,
5570                                                           item_flags,
5571                                                           error);
5572                         if (ret < 0)
5573                                 return ret;
5574                         /* Count all modify-header actions as one action. */
5575                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5576                                 ++actions_n;
5577                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5578                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5579                         break;
5580                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5581                         ret = flow_dv_validate_action_modify_ipv6_dscp
5582                                                                 (action_flags,
5583                                                                  actions,
5584                                                                  item_flags,
5585                                                                  error);
5586                         if (ret < 0)
5587                                 return ret;
5588                         /* Count all modify-header actions as one action. */
5589                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5590                                 ++actions_n;
5591                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5592                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5593                         break;
5594                 default:
5595                         return rte_flow_error_set(error, ENOTSUP,
5596                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5597                                                   actions,
5598                                                   "action not supported");
5599                 }
5600         }
5601         /*
5602          * Validate the drop action mutual exclusion with other actions.
5603          * Drop action is mutually-exclusive with any other action, except for
5604          * Count action.
5605          */
5606         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
5607             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
5608                 return rte_flow_error_set(error, EINVAL,
5609                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5610                                           "Drop action is mutually-exclusive "
5611                                           "with any other action, except for "
5612                                           "Count action");
5613         /* Eswitch has few restrictions on using items and actions */
5614         if (attr->transfer) {
5615                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5616                     action_flags & MLX5_FLOW_ACTION_FLAG)
5617                         return rte_flow_error_set(error, ENOTSUP,
5618                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5619                                                   NULL,
5620                                                   "unsupported action FLAG");
5621                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5622                     action_flags & MLX5_FLOW_ACTION_MARK)
5623                         return rte_flow_error_set(error, ENOTSUP,
5624                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5625                                                   NULL,
5626                                                   "unsupported action MARK");
5627                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
5628                         return rte_flow_error_set(error, ENOTSUP,
5629                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5630                                                   NULL,
5631                                                   "unsupported action QUEUE");
5632                 if (action_flags & MLX5_FLOW_ACTION_RSS)
5633                         return rte_flow_error_set(error, ENOTSUP,
5634                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5635                                                   NULL,
5636                                                   "unsupported action RSS");
5637                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5638                         return rte_flow_error_set(error, EINVAL,
5639                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5640                                                   actions,
5641                                                   "no fate action is found");
5642         } else {
5643                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
5644                         return rte_flow_error_set(error, EINVAL,
5645                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5646                                                   actions,
5647                                                   "no fate action is found");
5648         }
5649         /* Continue validation for Xcap actions.*/
5650         if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) && (queue_index == 0xFFFF ||
5651             mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5652                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5653                     MLX5_FLOW_XCAP_ACTIONS)
5654                         return rte_flow_error_set(error, ENOTSUP,
5655                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5656                                                   NULL, "encap and decap "
5657                                                   "combination aren't supported");
5658                 if (!attr->transfer && attr->ingress && (action_flags &
5659                                                         MLX5_FLOW_ACTION_ENCAP))
5660                         return rte_flow_error_set(error, ENOTSUP,
5661                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5662                                                   NULL, "encap is not supported"
5663                                                   " for ingress traffic");
5664         }
5665         /* Hairpin flow will add one more TAG action. */
5666         if (hairpin > 0)
5667                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5668         /* extra metadata enabled: one more TAG action will be add. */
5669         if (dev_conf->dv_flow_en &&
5670             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
5671             mlx5_flow_ext_mreg_supported(dev))
5672                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5673         if ((uint32_t)rw_act_num >
5674                         flow_dv_modify_hdr_action_max(dev, is_root)) {
5675                 return rte_flow_error_set(error, ENOTSUP,
5676                                           RTE_FLOW_ERROR_TYPE_ACTION,
5677                                           NULL, "too many header modify"
5678                                           " actions to support");
5679         }
5680         return 0;
5681 }
5682
5683 /**
5684  * Internal preparation function. Allocates the DV flow size,
5685  * this size is constant.
5686  *
5687  * @param[in] dev
5688  *   Pointer to the rte_eth_dev structure.
5689  * @param[in] attr
5690  *   Pointer to the flow attributes.
5691  * @param[in] items
5692  *   Pointer to the list of items.
5693  * @param[in] actions
5694  *   Pointer to the list of actions.
5695  * @param[out] error
5696  *   Pointer to the error structure.
5697  *
5698  * @return
5699  *   Pointer to mlx5_flow object on success,
5700  *   otherwise NULL and rte_errno is set.
5701  */
5702 static struct mlx5_flow *
5703 flow_dv_prepare(struct rte_eth_dev *dev,
5704                 const struct rte_flow_attr *attr __rte_unused,
5705                 const struct rte_flow_item items[] __rte_unused,
5706                 const struct rte_flow_action actions[] __rte_unused,
5707                 struct rte_flow_error *error)
5708 {
5709         uint32_t handle_idx = 0;
5710         struct mlx5_flow *dev_flow;
5711         struct mlx5_flow_handle *dev_handle;
5712         struct mlx5_priv *priv = dev->data->dev_private;
5713
5714         /* In case of corrupting the memory. */
5715         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
5716                 rte_flow_error_set(error, ENOSPC,
5717                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5718                                    "not free temporary device flow");
5719                 return NULL;
5720         }
5721         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
5722                                    &handle_idx);
5723         if (!dev_handle) {
5724                 rte_flow_error_set(error, ENOMEM,
5725                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5726                                    "not enough memory to create flow handle");
5727                 return NULL;
5728         }
5729         /* No multi-thread supporting. */
5730         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
5731         dev_flow->handle = dev_handle;
5732         dev_flow->handle_idx = handle_idx;
5733         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
5734         /*
5735          * The matching value needs to be cleared to 0 before using. In the
5736          * past, it will be automatically cleared when using rte_*alloc
5737          * API. The time consumption will be almost the same as before.
5738          */
5739         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
5740         dev_flow->ingress = attr->ingress;
5741         dev_flow->dv.transfer = attr->transfer;
5742         return dev_flow;
5743 }
5744
5745 #ifdef RTE_LIBRTE_MLX5_DEBUG
5746 /**
5747  * Sanity check for match mask and value. Similar to check_valid_spec() in
5748  * kernel driver. If unmasked bit is present in value, it returns failure.
5749  *
5750  * @param match_mask
5751  *   pointer to match mask buffer.
5752  * @param match_value
5753  *   pointer to match value buffer.
5754  *
5755  * @return
5756  *   0 if valid, -EINVAL otherwise.
5757  */
5758 static int
5759 flow_dv_check_valid_spec(void *match_mask, void *match_value)
5760 {
5761         uint8_t *m = match_mask;
5762         uint8_t *v = match_value;
5763         unsigned int i;
5764
5765         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
5766                 if (v[i] & ~m[i]) {
5767                         DRV_LOG(ERR,
5768                                 "match_value differs from match_criteria"
5769                                 " %p[%u] != %p[%u]",
5770                                 match_value, i, match_mask, i);
5771                         return -EINVAL;
5772                 }
5773         }
5774         return 0;
5775 }
5776 #endif
5777
5778 /**
5779  * Add match of ip_version.
5780  *
5781  * @param[in] group
5782  *   Flow group.
5783  * @param[in] headers_v
5784  *   Values header pointer.
5785  * @param[in] headers_m
5786  *   Masks header pointer.
5787  * @param[in] ip_version
5788  *   The IP version to set.
5789  */
5790 static inline void
5791 flow_dv_set_match_ip_version(uint32_t group,
5792                              void *headers_v,
5793                              void *headers_m,
5794                              uint8_t ip_version)
5795 {
5796         if (group == 0)
5797                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
5798         else
5799                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
5800                          ip_version);
5801         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
5802         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
5803         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
5804 }
5805
5806 /**
5807  * Add Ethernet item to matcher and to the value.
5808  *
5809  * @param[in, out] matcher
5810  *   Flow matcher.
5811  * @param[in, out] key
5812  *   Flow matcher value.
5813  * @param[in] item
5814  *   Flow pattern to translate.
5815  * @param[in] inner
5816  *   Item is inner pattern.
5817  */
5818 static void
5819 flow_dv_translate_item_eth(void *matcher, void *key,
5820                            const struct rte_flow_item *item, int inner,
5821                            uint32_t group)
5822 {
5823         const struct rte_flow_item_eth *eth_m = item->mask;
5824         const struct rte_flow_item_eth *eth_v = item->spec;
5825         const struct rte_flow_item_eth nic_mask = {
5826                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5827                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5828                 .type = RTE_BE16(0xffff),
5829         };
5830         void *headers_m;
5831         void *headers_v;
5832         char *l24_v;
5833         unsigned int i;
5834
5835         if (!eth_v)
5836                 return;
5837         if (!eth_m)
5838                 eth_m = &nic_mask;
5839         if (inner) {
5840                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5841                                          inner_headers);
5842                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5843         } else {
5844                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5845                                          outer_headers);
5846                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5847         }
5848         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
5849                &eth_m->dst, sizeof(eth_m->dst));
5850         /* The value must be in the range of the mask. */
5851         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
5852         for (i = 0; i < sizeof(eth_m->dst); ++i)
5853                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
5854         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
5855                &eth_m->src, sizeof(eth_m->src));
5856         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
5857         /* The value must be in the range of the mask. */
5858         for (i = 0; i < sizeof(eth_m->dst); ++i)
5859                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
5860         if (eth_v->type) {
5861                 /* When ethertype is present set mask for tagged VLAN. */
5862                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5863                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
5864                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
5865                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
5866                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
5867                                  1);
5868                         /* Return here to avoid setting match on ethertype. */
5869                         return;
5870                 }
5871         }
5872         /*
5873          * HW supports match on one Ethertype, the Ethertype following the last
5874          * VLAN tag of the packet (see PRM).
5875          * Set match on ethertype only if ETH header is not followed by VLAN.
5876          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5877          * ethertype, and use ip_version field instead.
5878          */
5879         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5880             eth_m->type == 0xFFFF) {
5881                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5882         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5883                    eth_m->type == 0xFFFF) {
5884                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5885         } else {
5886                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5887                          rte_be_to_cpu_16(eth_m->type));
5888                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5889                                      ethertype);
5890                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
5891         }
5892 }
5893
5894 /**
5895  * Add VLAN item to matcher and to the value.
5896  *
5897  * @param[in, out] dev_flow
5898  *   Flow descriptor.
5899  * @param[in, out] matcher
5900  *   Flow matcher.
5901  * @param[in, out] key
5902  *   Flow matcher value.
5903  * @param[in] item
5904  *   Flow pattern to translate.
5905  * @param[in] inner
5906  *   Item is inner pattern.
5907  */
5908 static void
5909 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
5910                             void *matcher, void *key,
5911                             const struct rte_flow_item *item,
5912                             int inner, uint32_t group)
5913 {
5914         const struct rte_flow_item_vlan *vlan_m = item->mask;
5915         const struct rte_flow_item_vlan *vlan_v = item->spec;
5916         void *headers_m;
5917         void *headers_v;
5918         uint16_t tci_m;
5919         uint16_t tci_v;
5920
5921         if (inner) {
5922                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5923                                          inner_headers);
5924                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5925         } else {
5926                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5927                                          outer_headers);
5928                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5929                 /*
5930                  * This is workaround, masks are not supported,
5931                  * and pre-validated.
5932                  */
5933                 if (vlan_v)
5934                         dev_flow->handle->vf_vlan.tag =
5935                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
5936         }
5937         /*
5938          * When VLAN item exists in flow, mark packet as tagged,
5939          * even if TCI is not specified.
5940          */
5941         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5942         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
5943         if (!vlan_v)
5944                 return;
5945         if (!vlan_m)
5946                 vlan_m = &rte_flow_item_vlan_mask;
5947         tci_m = rte_be_to_cpu_16(vlan_m->tci);
5948         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
5949         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
5950         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
5951         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
5952         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
5953         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
5954         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
5955         /*
5956          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5957          * ethertype, and use ip_version field instead.
5958          */
5959         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5960             vlan_m->inner_type == 0xFFFF) {
5961                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5962         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5963                    vlan_m->inner_type == 0xFFFF) {
5964                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5965         } else {
5966                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5967                          rte_be_to_cpu_16(vlan_m->inner_type));
5968                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
5969                          rte_be_to_cpu_16(vlan_m->inner_type &
5970                                           vlan_v->inner_type));
5971         }
5972 }
5973
5974 /**
5975  * Add IPV4 item to matcher and to the value.
5976  *
5977  * @param[in, out] matcher
5978  *   Flow matcher.
5979  * @param[in, out] key
5980  *   Flow matcher value.
5981  * @param[in] item
5982  *   Flow pattern to translate.
5983  * @param[in] item_flags
5984  *   Bit-fields that holds the items detected until now.
5985  * @param[in] inner
5986  *   Item is inner pattern.
5987  * @param[in] group
5988  *   The group to insert the rule.
5989  */
5990 static void
5991 flow_dv_translate_item_ipv4(void *matcher, void *key,
5992                             const struct rte_flow_item *item,
5993                             const uint64_t item_flags,
5994                             int inner, uint32_t group)
5995 {
5996         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
5997         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
5998         const struct rte_flow_item_ipv4 nic_mask = {
5999                 .hdr = {
6000                         .src_addr = RTE_BE32(0xffffffff),
6001                         .dst_addr = RTE_BE32(0xffffffff),
6002                         .type_of_service = 0xff,
6003                         .next_proto_id = 0xff,
6004                         .time_to_live = 0xff,
6005                 },
6006         };
6007         void *headers_m;
6008         void *headers_v;
6009         char *l24_m;
6010         char *l24_v;
6011         uint8_t tos;
6012
6013         if (inner) {
6014                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6015                                          inner_headers);
6016                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6017         } else {
6018                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6019                                          outer_headers);
6020                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6021         }
6022         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6023         /*
6024          * On outer header (which must contains L2), or inner header with L2,
6025          * set cvlan_tag mask bit to mark this packet as untagged.
6026          * This should be done even if item->spec is empty.
6027          */
6028         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6029                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6030         if (!ipv4_v)
6031                 return;
6032         if (!ipv4_m)
6033                 ipv4_m = &nic_mask;
6034         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6035                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6036         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6037                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6038         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6039         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6040         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6041                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6042         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6043                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6044         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6045         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6046         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6047         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6048                  ipv4_m->hdr.type_of_service);
6049         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6050         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6051                  ipv4_m->hdr.type_of_service >> 2);
6052         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6053         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6054                  ipv4_m->hdr.next_proto_id);
6055         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6056                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6057         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6058                  ipv4_m->hdr.time_to_live);
6059         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6060                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6061 }
6062
6063 /**
6064  * Add IPV6 item to matcher and to the value.
6065  *
6066  * @param[in, out] matcher
6067  *   Flow matcher.
6068  * @param[in, out] key
6069  *   Flow matcher value.
6070  * @param[in] item
6071  *   Flow pattern to translate.
6072  * @param[in] item_flags
6073  *   Bit-fields that holds the items detected until now.
6074  * @param[in] inner
6075  *   Item is inner pattern.
6076  * @param[in] group
6077  *   The group to insert the rule.
6078  */
6079 static void
6080 flow_dv_translate_item_ipv6(void *matcher, void *key,
6081                             const struct rte_flow_item *item,
6082                             const uint64_t item_flags,
6083                             int inner, uint32_t group)
6084 {
6085         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6086         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6087         const struct rte_flow_item_ipv6 nic_mask = {
6088                 .hdr = {
6089                         .src_addr =
6090                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6091                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6092                         .dst_addr =
6093                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6094                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6095                         .vtc_flow = RTE_BE32(0xffffffff),
6096                         .proto = 0xff,
6097                         .hop_limits = 0xff,
6098                 },
6099         };
6100         void *headers_m;
6101         void *headers_v;
6102         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6103         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6104         char *l24_m;
6105         char *l24_v;
6106         uint32_t vtc_m;
6107         uint32_t vtc_v;
6108         int i;
6109         int size;
6110
6111         if (inner) {
6112                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6113                                          inner_headers);
6114                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6115         } else {
6116                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6117                                          outer_headers);
6118                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6119         }
6120         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6121         /*
6122          * On outer header (which must contains L2), or inner header with L2,
6123          * set cvlan_tag mask bit to mark this packet as untagged.
6124          * This should be done even if item->spec is empty.
6125          */
6126         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6127                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6128         if (!ipv6_v)
6129                 return;
6130         if (!ipv6_m)
6131                 ipv6_m = &nic_mask;
6132         size = sizeof(ipv6_m->hdr.dst_addr);
6133         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6134                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6135         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6136                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6137         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6138         for (i = 0; i < size; ++i)
6139                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6140         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6141                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6142         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6143                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6144         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6145         for (i = 0; i < size; ++i)
6146                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6147         /* TOS. */
6148         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6149         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6150         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6151         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6152         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6153         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6154         /* Label. */
6155         if (inner) {
6156                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6157                          vtc_m);
6158                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6159                          vtc_v);
6160         } else {
6161                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6162                          vtc_m);
6163                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6164                          vtc_v);
6165         }
6166         /* Protocol. */
6167         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6168                  ipv6_m->hdr.proto);
6169         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6170                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6171         /* Hop limit. */
6172         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6173                  ipv6_m->hdr.hop_limits);
6174         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6175                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6176 }
6177
6178 /**
6179  * Add TCP item to matcher and to the value.
6180  *
6181  * @param[in, out] matcher
6182  *   Flow matcher.
6183  * @param[in, out] key
6184  *   Flow matcher value.
6185  * @param[in] item
6186  *   Flow pattern to translate.
6187  * @param[in] inner
6188  *   Item is inner pattern.
6189  */
6190 static void
6191 flow_dv_translate_item_tcp(void *matcher, void *key,
6192                            const struct rte_flow_item *item,
6193                            int inner)
6194 {
6195         const struct rte_flow_item_tcp *tcp_m = item->mask;
6196         const struct rte_flow_item_tcp *tcp_v = item->spec;
6197         void *headers_m;
6198         void *headers_v;
6199
6200         if (inner) {
6201                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6202                                          inner_headers);
6203                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6204         } else {
6205                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6206                                          outer_headers);
6207                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6208         }
6209         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6210         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6211         if (!tcp_v)
6212                 return;
6213         if (!tcp_m)
6214                 tcp_m = &rte_flow_item_tcp_mask;
6215         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6216                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6217         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6218                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6219         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6220                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6221         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6222                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6223         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6224                  tcp_m->hdr.tcp_flags);
6225         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6226                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6227 }
6228
6229 /**
6230  * Add UDP item to matcher and to the value.
6231  *
6232  * @param[in, out] matcher
6233  *   Flow matcher.
6234  * @param[in, out] key
6235  *   Flow matcher value.
6236  * @param[in] item
6237  *   Flow pattern to translate.
6238  * @param[in] inner
6239  *   Item is inner pattern.
6240  */
6241 static void
6242 flow_dv_translate_item_udp(void *matcher, void *key,
6243                            const struct rte_flow_item *item,
6244                            int inner)
6245 {
6246         const struct rte_flow_item_udp *udp_m = item->mask;
6247         const struct rte_flow_item_udp *udp_v = item->spec;
6248         void *headers_m;
6249         void *headers_v;
6250
6251         if (inner) {
6252                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6253                                          inner_headers);
6254                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6255         } else {
6256                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6257                                          outer_headers);
6258                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6259         }
6260         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6261         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6262         if (!udp_v)
6263                 return;
6264         if (!udp_m)
6265                 udp_m = &rte_flow_item_udp_mask;
6266         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6267                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6268         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6269                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6270         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6271                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6272         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6273                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6274 }
6275
6276 /**
6277  * Add GRE optional Key item to matcher and to the value.
6278  *
6279  * @param[in, out] matcher
6280  *   Flow matcher.
6281  * @param[in, out] key
6282  *   Flow matcher value.
6283  * @param[in] item
6284  *   Flow pattern to translate.
6285  * @param[in] inner
6286  *   Item is inner pattern.
6287  */
6288 static void
6289 flow_dv_translate_item_gre_key(void *matcher, void *key,
6290                                    const struct rte_flow_item *item)
6291 {
6292         const rte_be32_t *key_m = item->mask;
6293         const rte_be32_t *key_v = item->spec;
6294         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6295         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6296         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6297
6298         /* GRE K bit must be on and should already be validated */
6299         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6300         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6301         if (!key_v)
6302                 return;
6303         if (!key_m)
6304                 key_m = &gre_key_default_mask;
6305         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6306                  rte_be_to_cpu_32(*key_m) >> 8);
6307         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6308                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6309         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6310                  rte_be_to_cpu_32(*key_m) & 0xFF);
6311         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6312                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6313 }
6314
6315 /**
6316  * Add GRE item to matcher and to the value.
6317  *
6318  * @param[in, out] matcher
6319  *   Flow matcher.
6320  * @param[in, out] key
6321  *   Flow matcher value.
6322  * @param[in] item
6323  *   Flow pattern to translate.
6324  * @param[in] inner
6325  *   Item is inner pattern.
6326  */
6327 static void
6328 flow_dv_translate_item_gre(void *matcher, void *key,
6329                            const struct rte_flow_item *item,
6330                            int inner)
6331 {
6332         const struct rte_flow_item_gre *gre_m = item->mask;
6333         const struct rte_flow_item_gre *gre_v = item->spec;
6334         void *headers_m;
6335         void *headers_v;
6336         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6337         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6338         struct {
6339                 union {
6340                         __extension__
6341                         struct {
6342                                 uint16_t version:3;
6343                                 uint16_t rsvd0:9;
6344                                 uint16_t s_present:1;
6345                                 uint16_t k_present:1;
6346                                 uint16_t rsvd_bit1:1;
6347                                 uint16_t c_present:1;
6348                         };
6349                         uint16_t value;
6350                 };
6351         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6352
6353         if (inner) {
6354                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6355                                          inner_headers);
6356                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6357         } else {
6358                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6359                                          outer_headers);
6360                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6361         }
6362         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6363         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6364         if (!gre_v)
6365                 return;
6366         if (!gre_m)
6367                 gre_m = &rte_flow_item_gre_mask;
6368         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6369                  rte_be_to_cpu_16(gre_m->protocol));
6370         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6371                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6372         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6373         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6374         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6375                  gre_crks_rsvd0_ver_m.c_present);
6376         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6377                  gre_crks_rsvd0_ver_v.c_present &
6378                  gre_crks_rsvd0_ver_m.c_present);
6379         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6380                  gre_crks_rsvd0_ver_m.k_present);
6381         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6382                  gre_crks_rsvd0_ver_v.k_present &
6383                  gre_crks_rsvd0_ver_m.k_present);
6384         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6385                  gre_crks_rsvd0_ver_m.s_present);
6386         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6387                  gre_crks_rsvd0_ver_v.s_present &
6388                  gre_crks_rsvd0_ver_m.s_present);
6389 }
6390
6391 /**
6392  * Add NVGRE item to matcher and to the value.
6393  *
6394  * @param[in, out] matcher
6395  *   Flow matcher.
6396  * @param[in, out] key
6397  *   Flow matcher value.
6398  * @param[in] item
6399  *   Flow pattern to translate.
6400  * @param[in] inner
6401  *   Item is inner pattern.
6402  */
6403 static void
6404 flow_dv_translate_item_nvgre(void *matcher, void *key,
6405                              const struct rte_flow_item *item,
6406                              int inner)
6407 {
6408         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6409         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6410         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6411         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6412         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
6413         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
6414         char *gre_key_m;
6415         char *gre_key_v;
6416         int size;
6417         int i;
6418
6419         /* For NVGRE, GRE header fields must be set with defined values. */
6420         const struct rte_flow_item_gre gre_spec = {
6421                 .c_rsvd0_ver = RTE_BE16(0x2000),
6422                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6423         };
6424         const struct rte_flow_item_gre gre_mask = {
6425                 .c_rsvd0_ver = RTE_BE16(0xB000),
6426                 .protocol = RTE_BE16(UINT16_MAX),
6427         };
6428         const struct rte_flow_item gre_item = {
6429                 .spec = &gre_spec,
6430                 .mask = &gre_mask,
6431                 .last = NULL,
6432         };
6433         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6434         if (!nvgre_v)
6435                 return;
6436         if (!nvgre_m)
6437                 nvgre_m = &rte_flow_item_nvgre_mask;
6438         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6439         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6440         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6441         memcpy(gre_key_m, tni_flow_id_m, size);
6442         for (i = 0; i < size; ++i)
6443                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6444 }
6445
6446 /**
6447  * Add VXLAN item to matcher and to the value.
6448  *
6449  * @param[in, out] matcher
6450  *   Flow matcher.
6451  * @param[in, out] key
6452  *   Flow matcher value.
6453  * @param[in] item
6454  *   Flow pattern to translate.
6455  * @param[in] inner
6456  *   Item is inner pattern.
6457  */
6458 static void
6459 flow_dv_translate_item_vxlan(void *matcher, void *key,
6460                              const struct rte_flow_item *item,
6461                              int inner)
6462 {
6463         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6464         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6465         void *headers_m;
6466         void *headers_v;
6467         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6468         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6469         char *vni_m;
6470         char *vni_v;
6471         uint16_t dport;
6472         int size;
6473         int i;
6474
6475         if (inner) {
6476                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6477                                          inner_headers);
6478                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6479         } else {
6480                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6481                                          outer_headers);
6482                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6483         }
6484         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6485                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6486         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6487                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6488                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6489         }
6490         if (!vxlan_v)
6491                 return;
6492         if (!vxlan_m)
6493                 vxlan_m = &rte_flow_item_vxlan_mask;
6494         size = sizeof(vxlan_m->vni);
6495         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6496         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6497         memcpy(vni_m, vxlan_m->vni, size);
6498         for (i = 0; i < size; ++i)
6499                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6500 }
6501
6502 /**
6503  * Add VXLAN-GPE item to matcher and to the value.
6504  *
6505  * @param[in, out] matcher
6506  *   Flow matcher.
6507  * @param[in, out] key
6508  *   Flow matcher value.
6509  * @param[in] item
6510  *   Flow pattern to translate.
6511  * @param[in] inner
6512  *   Item is inner pattern.
6513  */
6514
6515 static void
6516 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
6517                                  const struct rte_flow_item *item, int inner)
6518 {
6519         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
6520         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
6521         void *headers_m;
6522         void *headers_v;
6523         void *misc_m =
6524                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
6525         void *misc_v =
6526                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6527         char *vni_m;
6528         char *vni_v;
6529         uint16_t dport;
6530         int size;
6531         int i;
6532         uint8_t flags_m = 0xff;
6533         uint8_t flags_v = 0xc;
6534
6535         if (inner) {
6536                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6537                                          inner_headers);
6538                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6539         } else {
6540                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6541                                          outer_headers);
6542                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6543         }
6544         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6545                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6546         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6547                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6548                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6549         }
6550         if (!vxlan_v)
6551                 return;
6552         if (!vxlan_m)
6553                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
6554         size = sizeof(vxlan_m->vni);
6555         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
6556         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
6557         memcpy(vni_m, vxlan_m->vni, size);
6558         for (i = 0; i < size; ++i)
6559                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6560         if (vxlan_m->flags) {
6561                 flags_m = vxlan_m->flags;
6562                 flags_v = vxlan_v->flags;
6563         }
6564         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
6565         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
6566         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
6567                  vxlan_m->protocol);
6568         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
6569                  vxlan_v->protocol);
6570 }
6571
6572 /**
6573  * Add Geneve item to matcher and to the value.
6574  *
6575  * @param[in, out] matcher
6576  *   Flow matcher.
6577  * @param[in, out] key
6578  *   Flow matcher value.
6579  * @param[in] item
6580  *   Flow pattern to translate.
6581  * @param[in] inner
6582  *   Item is inner pattern.
6583  */
6584
6585 static void
6586 flow_dv_translate_item_geneve(void *matcher, void *key,
6587                               const struct rte_flow_item *item, int inner)
6588 {
6589         const struct rte_flow_item_geneve *geneve_m = item->mask;
6590         const struct rte_flow_item_geneve *geneve_v = item->spec;
6591         void *headers_m;
6592         void *headers_v;
6593         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6594         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6595         uint16_t dport;
6596         uint16_t gbhdr_m;
6597         uint16_t gbhdr_v;
6598         char *vni_m;
6599         char *vni_v;
6600         size_t size, i;
6601
6602         if (inner) {
6603                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6604                                          inner_headers);
6605                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6606         } else {
6607                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6608                                          outer_headers);
6609                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6610         }
6611         dport = MLX5_UDP_PORT_GENEVE;
6612         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6613                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6614                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6615         }
6616         if (!geneve_v)
6617                 return;
6618         if (!geneve_m)
6619                 geneve_m = &rte_flow_item_geneve_mask;
6620         size = sizeof(geneve_m->vni);
6621         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
6622         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
6623         memcpy(vni_m, geneve_m->vni, size);
6624         for (i = 0; i < size; ++i)
6625                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
6626         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
6627                  rte_be_to_cpu_16(geneve_m->protocol));
6628         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
6629                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
6630         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
6631         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
6632         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
6633                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6634         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
6635                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6636         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
6637                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6638         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
6639                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
6640                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6641 }
6642
6643 /**
6644  * Add MPLS item to matcher and to the value.
6645  *
6646  * @param[in, out] matcher
6647  *   Flow matcher.
6648  * @param[in, out] key
6649  *   Flow matcher value.
6650  * @param[in] item
6651  *   Flow pattern to translate.
6652  * @param[in] prev_layer
6653  *   The protocol layer indicated in previous item.
6654  * @param[in] inner
6655  *   Item is inner pattern.
6656  */
6657 static void
6658 flow_dv_translate_item_mpls(void *matcher, void *key,
6659                             const struct rte_flow_item *item,
6660                             uint64_t prev_layer,
6661                             int inner)
6662 {
6663         const uint32_t *in_mpls_m = item->mask;
6664         const uint32_t *in_mpls_v = item->spec;
6665         uint32_t *out_mpls_m = 0;
6666         uint32_t *out_mpls_v = 0;
6667         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6668         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6669         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
6670                                      misc_parameters_2);
6671         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6672         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
6673         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6674
6675         switch (prev_layer) {
6676         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6677                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
6678                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6679                          MLX5_UDP_PORT_MPLS);
6680                 break;
6681         case MLX5_FLOW_LAYER_GRE:
6682                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
6683                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6684                          RTE_ETHER_TYPE_MPLS);
6685                 break;
6686         default:
6687                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6688                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6689                          IPPROTO_MPLS);
6690                 break;
6691         }
6692         if (!in_mpls_v)
6693                 return;
6694         if (!in_mpls_m)
6695                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
6696         switch (prev_layer) {
6697         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6698                 out_mpls_m =
6699                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6700                                                  outer_first_mpls_over_udp);
6701                 out_mpls_v =
6702                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6703                                                  outer_first_mpls_over_udp);
6704                 break;
6705         case MLX5_FLOW_LAYER_GRE:
6706                 out_mpls_m =
6707                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6708                                                  outer_first_mpls_over_gre);
6709                 out_mpls_v =
6710                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6711                                                  outer_first_mpls_over_gre);
6712                 break;
6713         default:
6714                 /* Inner MPLS not over GRE is not supported. */
6715                 if (!inner) {
6716                         out_mpls_m =
6717                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6718                                                          misc2_m,
6719                                                          outer_first_mpls);
6720                         out_mpls_v =
6721                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6722                                                          misc2_v,
6723                                                          outer_first_mpls);
6724                 }
6725                 break;
6726         }
6727         if (out_mpls_m && out_mpls_v) {
6728                 *out_mpls_m = *in_mpls_m;
6729                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
6730         }
6731 }
6732
6733 /**
6734  * Add metadata register item to matcher
6735  *
6736  * @param[in, out] matcher
6737  *   Flow matcher.
6738  * @param[in, out] key
6739  *   Flow matcher value.
6740  * @param[in] reg_type
6741  *   Type of device metadata register
6742  * @param[in] value
6743  *   Register value
6744  * @param[in] mask
6745  *   Register mask
6746  */
6747 static void
6748 flow_dv_match_meta_reg(void *matcher, void *key,
6749                        enum modify_reg reg_type,
6750                        uint32_t data, uint32_t mask)
6751 {
6752         void *misc2_m =
6753                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
6754         void *misc2_v =
6755                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6756         uint32_t temp;
6757
6758         data &= mask;
6759         switch (reg_type) {
6760         case REG_A:
6761                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
6762                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
6763                 break;
6764         case REG_B:
6765                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
6766                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
6767                 break;
6768         case REG_C_0:
6769                 /*
6770                  * The metadata register C0 field might be divided into
6771                  * source vport index and META item value, we should set
6772                  * this field according to specified mask, not as whole one.
6773                  */
6774                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
6775                 temp |= mask;
6776                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
6777                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
6778                 temp &= ~mask;
6779                 temp |= data;
6780                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
6781                 break;
6782         case REG_C_1:
6783                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
6784                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
6785                 break;
6786         case REG_C_2:
6787                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
6788                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
6789                 break;
6790         case REG_C_3:
6791                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
6792                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
6793                 break;
6794         case REG_C_4:
6795                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
6796                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
6797                 break;
6798         case REG_C_5:
6799                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
6800                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
6801                 break;
6802         case REG_C_6:
6803                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
6804                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
6805                 break;
6806         case REG_C_7:
6807                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
6808                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
6809                 break;
6810         default:
6811                 MLX5_ASSERT(false);
6812                 break;
6813         }
6814 }
6815
6816 /**
6817  * Add MARK item to matcher
6818  *
6819  * @param[in] dev
6820  *   The device to configure through.
6821  * @param[in, out] matcher
6822  *   Flow matcher.
6823  * @param[in, out] key
6824  *   Flow matcher value.
6825  * @param[in] item
6826  *   Flow pattern to translate.
6827  */
6828 static void
6829 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
6830                             void *matcher, void *key,
6831                             const struct rte_flow_item *item)
6832 {
6833         struct mlx5_priv *priv = dev->data->dev_private;
6834         const struct rte_flow_item_mark *mark;
6835         uint32_t value;
6836         uint32_t mask;
6837
6838         mark = item->mask ? (const void *)item->mask :
6839                             &rte_flow_item_mark_mask;
6840         mask = mark->id & priv->sh->dv_mark_mask;
6841         mark = (const void *)item->spec;
6842         MLX5_ASSERT(mark);
6843         value = mark->id & priv->sh->dv_mark_mask & mask;
6844         if (mask) {
6845                 enum modify_reg reg;
6846
6847                 /* Get the metadata register index for the mark. */
6848                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
6849                 MLX5_ASSERT(reg > 0);
6850                 if (reg == REG_C_0) {
6851                         struct mlx5_priv *priv = dev->data->dev_private;
6852                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6853                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6854
6855                         mask &= msk_c0;
6856                         mask <<= shl_c0;
6857                         value <<= shl_c0;
6858                 }
6859                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6860         }
6861 }
6862
6863 /**
6864  * Add META item to matcher
6865  *
6866  * @param[in] dev
6867  *   The devich to configure through.
6868  * @param[in, out] matcher
6869  *   Flow matcher.
6870  * @param[in, out] key
6871  *   Flow matcher value.
6872  * @param[in] attr
6873  *   Attributes of flow that includes this item.
6874  * @param[in] item
6875  *   Flow pattern to translate.
6876  */
6877 static void
6878 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
6879                             void *matcher, void *key,
6880                             const struct rte_flow_attr *attr,
6881                             const struct rte_flow_item *item)
6882 {
6883         const struct rte_flow_item_meta *meta_m;
6884         const struct rte_flow_item_meta *meta_v;
6885
6886         meta_m = (const void *)item->mask;
6887         if (!meta_m)
6888                 meta_m = &rte_flow_item_meta_mask;
6889         meta_v = (const void *)item->spec;
6890         if (meta_v) {
6891                 int reg;
6892                 uint32_t value = meta_v->data;
6893                 uint32_t mask = meta_m->data;
6894
6895                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
6896                 if (reg < 0)
6897                         return;
6898                 /*
6899                  * In datapath code there is no endianness
6900                  * coversions for perfromance reasons, all
6901                  * pattern conversions are done in rte_flow.
6902                  */
6903                 value = rte_cpu_to_be_32(value);
6904                 mask = rte_cpu_to_be_32(mask);
6905                 if (reg == REG_C_0) {
6906                         struct mlx5_priv *priv = dev->data->dev_private;
6907                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6908                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6909 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6910                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
6911
6912                         value >>= shr_c0;
6913                         mask >>= shr_c0;
6914 #endif
6915                         value <<= shl_c0;
6916                         mask <<= shl_c0;
6917                         MLX5_ASSERT(msk_c0);
6918                         MLX5_ASSERT(!(~msk_c0 & mask));
6919                 }
6920                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6921         }
6922 }
6923
6924 /**
6925  * Add vport metadata Reg C0 item to matcher
6926  *
6927  * @param[in, out] matcher
6928  *   Flow matcher.
6929  * @param[in, out] key
6930  *   Flow matcher value.
6931  * @param[in] reg
6932  *   Flow pattern to translate.
6933  */
6934 static void
6935 flow_dv_translate_item_meta_vport(void *matcher, void *key,
6936                                   uint32_t value, uint32_t mask)
6937 {
6938         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
6939 }
6940
6941 /**
6942  * Add tag item to matcher
6943  *
6944  * @param[in] dev
6945  *   The devich to configure through.
6946  * @param[in, out] matcher
6947  *   Flow matcher.
6948  * @param[in, out] key
6949  *   Flow matcher value.
6950  * @param[in] item
6951  *   Flow pattern to translate.
6952  */
6953 static void
6954 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
6955                                 void *matcher, void *key,
6956                                 const struct rte_flow_item *item)
6957 {
6958         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
6959         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
6960         uint32_t mask, value;
6961
6962         MLX5_ASSERT(tag_v);
6963         value = tag_v->data;
6964         mask = tag_m ? tag_m->data : UINT32_MAX;
6965         if (tag_v->id == REG_C_0) {
6966                 struct mlx5_priv *priv = dev->data->dev_private;
6967                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6968                 uint32_t shl_c0 = rte_bsf32(msk_c0);
6969
6970                 mask &= msk_c0;
6971                 mask <<= shl_c0;
6972                 value <<= shl_c0;
6973         }
6974         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
6975 }
6976
6977 /**
6978  * Add TAG item to matcher
6979  *
6980  * @param[in] dev
6981  *   The devich to configure through.
6982  * @param[in, out] matcher
6983  *   Flow matcher.
6984  * @param[in, out] key
6985  *   Flow matcher value.
6986  * @param[in] item
6987  *   Flow pattern to translate.
6988  */
6989 static void
6990 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
6991                            void *matcher, void *key,
6992                            const struct rte_flow_item *item)
6993 {
6994         const struct rte_flow_item_tag *tag_v = item->spec;
6995         const struct rte_flow_item_tag *tag_m = item->mask;
6996         enum modify_reg reg;
6997
6998         MLX5_ASSERT(tag_v);
6999         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7000         /* Get the metadata register index for the tag. */
7001         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7002         MLX5_ASSERT(reg > 0);
7003         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7004 }
7005
7006 /**
7007  * Add source vport match to the specified matcher.
7008  *
7009  * @param[in, out] matcher
7010  *   Flow matcher.
7011  * @param[in, out] key
7012  *   Flow matcher value.
7013  * @param[in] port
7014  *   Source vport value to match
7015  * @param[in] mask
7016  *   Mask
7017  */
7018 static void
7019 flow_dv_translate_item_source_vport(void *matcher, void *key,
7020                                     int16_t port, uint16_t mask)
7021 {
7022         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7023         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7024
7025         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7026         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7027 }
7028
7029 /**
7030  * Translate port-id item to eswitch match on  port-id.
7031  *
7032  * @param[in] dev
7033  *   The devich to configure through.
7034  * @param[in, out] matcher
7035  *   Flow matcher.
7036  * @param[in, out] key
7037  *   Flow matcher value.
7038  * @param[in] item
7039  *   Flow pattern to translate.
7040  *
7041  * @return
7042  *   0 on success, a negative errno value otherwise.
7043  */
7044 static int
7045 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7046                                void *key, const struct rte_flow_item *item)
7047 {
7048         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7049         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7050         struct mlx5_priv *priv;
7051         uint16_t mask, id;
7052
7053         mask = pid_m ? pid_m->id : 0xffff;
7054         id = pid_v ? pid_v->id : dev->data->port_id;
7055         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7056         if (!priv)
7057                 return -rte_errno;
7058         /* Translate to vport field or to metadata, depending on mode. */
7059         if (priv->vport_meta_mask)
7060                 flow_dv_translate_item_meta_vport(matcher, key,
7061                                                   priv->vport_meta_tag,
7062                                                   priv->vport_meta_mask);
7063         else
7064                 flow_dv_translate_item_source_vport(matcher, key,
7065                                                     priv->vport_id, mask);
7066         return 0;
7067 }
7068
7069 /**
7070  * Add ICMP6 item to matcher and to the value.
7071  *
7072  * @param[in, out] matcher
7073  *   Flow matcher.
7074  * @param[in, out] key
7075  *   Flow matcher value.
7076  * @param[in] item
7077  *   Flow pattern to translate.
7078  * @param[in] inner
7079  *   Item is inner pattern.
7080  */
7081 static void
7082 flow_dv_translate_item_icmp6(void *matcher, void *key,
7083                               const struct rte_flow_item *item,
7084                               int inner)
7085 {
7086         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7087         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7088         void *headers_m;
7089         void *headers_v;
7090         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7091                                      misc_parameters_3);
7092         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7093         if (inner) {
7094                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7095                                          inner_headers);
7096                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7097         } else {
7098                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7099                                          outer_headers);
7100                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7101         }
7102         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7103         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7104         if (!icmp6_v)
7105                 return;
7106         if (!icmp6_m)
7107                 icmp6_m = &rte_flow_item_icmp6_mask;
7108         /*
7109          * Force flow only to match the non-fragmented IPv6 ICMPv6 packets.
7110          * If only the protocol is specified, no need to match the frag.
7111          */
7112         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7113         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7114         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7115         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7116                  icmp6_v->type & icmp6_m->type);
7117         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7118         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7119                  icmp6_v->code & icmp6_m->code);
7120 }
7121
7122 /**
7123  * Add ICMP item to matcher and to the value.
7124  *
7125  * @param[in, out] matcher
7126  *   Flow matcher.
7127  * @param[in, out] key
7128  *   Flow matcher value.
7129  * @param[in] item
7130  *   Flow pattern to translate.
7131  * @param[in] inner
7132  *   Item is inner pattern.
7133  */
7134 static void
7135 flow_dv_translate_item_icmp(void *matcher, void *key,
7136                             const struct rte_flow_item *item,
7137                             int inner)
7138 {
7139         const struct rte_flow_item_icmp *icmp_m = item->mask;
7140         const struct rte_flow_item_icmp *icmp_v = item->spec;
7141         void *headers_m;
7142         void *headers_v;
7143         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7144                                      misc_parameters_3);
7145         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7146         if (inner) {
7147                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7148                                          inner_headers);
7149                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7150         } else {
7151                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7152                                          outer_headers);
7153                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7154         }
7155         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7156         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7157         if (!icmp_v)
7158                 return;
7159         if (!icmp_m)
7160                 icmp_m = &rte_flow_item_icmp_mask;
7161         /*
7162          * Force flow only to match the non-fragmented IPv4 ICMP packets.
7163          * If only the protocol is specified, no need to match the frag.
7164          */
7165         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7166         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7167         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7168                  icmp_m->hdr.icmp_type);
7169         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7170                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7171         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7172                  icmp_m->hdr.icmp_code);
7173         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7174                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7175 }
7176
7177 /**
7178  * Add GTP item to matcher and to the value.
7179  *
7180  * @param[in, out] matcher
7181  *   Flow matcher.
7182  * @param[in, out] key
7183  *   Flow matcher value.
7184  * @param[in] item
7185  *   Flow pattern to translate.
7186  * @param[in] inner
7187  *   Item is inner pattern.
7188  */
7189 static void
7190 flow_dv_translate_item_gtp(void *matcher, void *key,
7191                            const struct rte_flow_item *item, int inner)
7192 {
7193         const struct rte_flow_item_gtp *gtp_m = item->mask;
7194         const struct rte_flow_item_gtp *gtp_v = item->spec;
7195         void *headers_m;
7196         void *headers_v;
7197         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7198                                      misc_parameters_3);
7199         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7200         uint16_t dport = RTE_GTPU_UDP_PORT;
7201
7202         if (inner) {
7203                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7204                                          inner_headers);
7205                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7206         } else {
7207                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7208                                          outer_headers);
7209                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7210         }
7211         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7212                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7213                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7214         }
7215         if (!gtp_v)
7216                 return;
7217         if (!gtp_m)
7218                 gtp_m = &rte_flow_item_gtp_mask;
7219         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7220                  gtp_m->v_pt_rsv_flags);
7221         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7222                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7223         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7224         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7225                  gtp_v->msg_type & gtp_m->msg_type);
7226         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7227                  rte_be_to_cpu_32(gtp_m->teid));
7228         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7229                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7230 }
7231
7232 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7233
7234 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7235         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7236                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7237
7238 /**
7239  * Calculate flow matcher enable bitmap.
7240  *
7241  * @param match_criteria
7242  *   Pointer to flow matcher criteria.
7243  *
7244  * @return
7245  *   Bitmap of enabled fields.
7246  */
7247 static uint8_t
7248 flow_dv_matcher_enable(uint32_t *match_criteria)
7249 {
7250         uint8_t match_criteria_enable;
7251
7252         match_criteria_enable =
7253                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7254                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7255         match_criteria_enable |=
7256                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7257                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7258         match_criteria_enable |=
7259                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7260                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7261         match_criteria_enable |=
7262                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7263                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7264         match_criteria_enable |=
7265                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7266                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7267         return match_criteria_enable;
7268 }
7269
7270
7271 /**
7272  * Get a flow table.
7273  *
7274  * @param[in, out] dev
7275  *   Pointer to rte_eth_dev structure.
7276  * @param[in] table_id
7277  *   Table id to use.
7278  * @param[in] egress
7279  *   Direction of the table.
7280  * @param[in] transfer
7281  *   E-Switch or NIC flow.
7282  * @param[out] error
7283  *   pointer to error structure.
7284  *
7285  * @return
7286  *   Returns tables resource based on the index, NULL in case of failed.
7287  */
7288 static struct mlx5_flow_tbl_resource *
7289 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7290                          uint32_t table_id, uint8_t egress,
7291                          uint8_t transfer,
7292                          struct rte_flow_error *error)
7293 {
7294         struct mlx5_priv *priv = dev->data->dev_private;
7295         struct mlx5_dev_ctx_shared *sh = priv->sh;
7296         struct mlx5_flow_tbl_resource *tbl;
7297         union mlx5_flow_tbl_key table_key = {
7298                 {
7299                         .table_id = table_id,
7300                         .reserved = 0,
7301                         .domain = !!transfer,
7302                         .direction = !!egress,
7303                 }
7304         };
7305         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7306                                                          table_key.v64);
7307         struct mlx5_flow_tbl_data_entry *tbl_data;
7308         uint32_t idx = 0;
7309         int ret;
7310         void *domain;
7311
7312         if (pos) {
7313                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7314                                         entry);
7315                 tbl = &tbl_data->tbl;
7316                 rte_atomic32_inc(&tbl->refcnt);
7317                 return tbl;
7318         }
7319         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7320         if (!tbl_data) {
7321                 rte_flow_error_set(error, ENOMEM,
7322                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7323                                    NULL,
7324                                    "cannot allocate flow table data entry");
7325                 return NULL;
7326         }
7327         tbl_data->idx = idx;
7328         tbl = &tbl_data->tbl;
7329         pos = &tbl_data->entry;
7330         if (transfer)
7331                 domain = sh->fdb_domain;
7332         else if (egress)
7333                 domain = sh->tx_domain;
7334         else
7335                 domain = sh->rx_domain;
7336         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
7337         if (ret) {
7338                 rte_flow_error_set(error, ENOMEM,
7339                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7340                                    NULL, "cannot create flow table object");
7341                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7342                 return NULL;
7343         }
7344         /*
7345          * No multi-threads now, but still better to initialize the reference
7346          * count before insert it into the hash list.
7347          */
7348         rte_atomic32_init(&tbl->refcnt);
7349         /* Jump action reference count is initialized here. */
7350         rte_atomic32_init(&tbl_data->jump.refcnt);
7351         pos->key = table_key.v64;
7352         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7353         if (ret < 0) {
7354                 rte_flow_error_set(error, -ret,
7355                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7356                                    "cannot insert flow table data entry");
7357                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7358                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7359         }
7360         rte_atomic32_inc(&tbl->refcnt);
7361         return tbl;
7362 }
7363
7364 /**
7365  * Release a flow table.
7366  *
7367  * @param[in] dev
7368  *   Pointer to rte_eth_dev structure.
7369  * @param[in] tbl
7370  *   Table resource to be released.
7371  *
7372  * @return
7373  *   Returns 0 if table was released, else return 1;
7374  */
7375 static int
7376 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7377                              struct mlx5_flow_tbl_resource *tbl)
7378 {
7379         struct mlx5_priv *priv = dev->data->dev_private;
7380         struct mlx5_dev_ctx_shared *sh = priv->sh;
7381         struct mlx5_flow_tbl_data_entry *tbl_data =
7382                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7383
7384         if (!tbl)
7385                 return 0;
7386         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7387                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7388
7389                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7390                 tbl->obj = NULL;
7391                 /* remove the entry from the hash list and free memory. */
7392                 mlx5_hlist_remove(sh->flow_tbls, pos);
7393                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7394                                 tbl_data->idx);
7395                 return 0;
7396         }
7397         return 1;
7398 }
7399
7400 /**
7401  * Register the flow matcher.
7402  *
7403  * @param[in, out] dev
7404  *   Pointer to rte_eth_dev structure.
7405  * @param[in, out] matcher
7406  *   Pointer to flow matcher.
7407  * @param[in, out] key
7408  *   Pointer to flow table key.
7409  * @parm[in, out] dev_flow
7410  *   Pointer to the dev_flow.
7411  * @param[out] error
7412  *   pointer to error structure.
7413  *
7414  * @return
7415  *   0 on success otherwise -errno and errno is set.
7416  */
7417 static int
7418 flow_dv_matcher_register(struct rte_eth_dev *dev,
7419                          struct mlx5_flow_dv_matcher *matcher,
7420                          union mlx5_flow_tbl_key *key,
7421                          struct mlx5_flow *dev_flow,
7422                          struct rte_flow_error *error)
7423 {
7424         struct mlx5_priv *priv = dev->data->dev_private;
7425         struct mlx5_dev_ctx_shared *sh = priv->sh;
7426         struct mlx5_flow_dv_matcher *cache_matcher;
7427         struct mlx5dv_flow_matcher_attr dv_attr = {
7428                 .type = IBV_FLOW_ATTR_NORMAL,
7429                 .match_mask = (void *)&matcher->mask,
7430         };
7431         struct mlx5_flow_tbl_resource *tbl;
7432         struct mlx5_flow_tbl_data_entry *tbl_data;
7433         int ret;
7434
7435         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
7436                                        key->domain, error);
7437         if (!tbl)
7438                 return -rte_errno;      /* No need to refill the error info */
7439         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7440         /* Lookup from cache. */
7441         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
7442                 if (matcher->crc == cache_matcher->crc &&
7443                     matcher->priority == cache_matcher->priority &&
7444                     !memcmp((const void *)matcher->mask.buf,
7445                             (const void *)cache_matcher->mask.buf,
7446                             cache_matcher->mask.size)) {
7447                         DRV_LOG(DEBUG,
7448                                 "%s group %u priority %hd use %s "
7449                                 "matcher %p: refcnt %d++",
7450                                 key->domain ? "FDB" : "NIC", key->table_id,
7451                                 cache_matcher->priority,
7452                                 key->direction ? "tx" : "rx",
7453                                 (void *)cache_matcher,
7454                                 rte_atomic32_read(&cache_matcher->refcnt));
7455                         rte_atomic32_inc(&cache_matcher->refcnt);
7456                         dev_flow->handle->dvh.matcher = cache_matcher;
7457                         /* old matcher should not make the table ref++. */
7458                         flow_dv_tbl_resource_release(dev, tbl);
7459                         return 0;
7460                 }
7461         }
7462         /* Register new matcher. */
7463         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
7464         if (!cache_matcher) {
7465                 flow_dv_tbl_resource_release(dev, tbl);
7466                 return rte_flow_error_set(error, ENOMEM,
7467                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7468                                           "cannot allocate matcher memory");
7469         }
7470         *cache_matcher = *matcher;
7471         dv_attr.match_criteria_enable =
7472                 flow_dv_matcher_enable(cache_matcher->mask.buf);
7473         dv_attr.priority = matcher->priority;
7474         if (key->direction)
7475                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
7476         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
7477                                                &cache_matcher->matcher_object);
7478         if (ret) {
7479                 rte_free(cache_matcher);
7480 #ifdef HAVE_MLX5DV_DR
7481                 flow_dv_tbl_resource_release(dev, tbl);
7482 #endif
7483                 return rte_flow_error_set(error, ENOMEM,
7484                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7485                                           NULL, "cannot create matcher");
7486         }
7487         /* Save the table information */
7488         cache_matcher->tbl = tbl;
7489         rte_atomic32_init(&cache_matcher->refcnt);
7490         /* only matcher ref++, table ref++ already done above in get API. */
7491         rte_atomic32_inc(&cache_matcher->refcnt);
7492         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
7493         dev_flow->handle->dvh.matcher = cache_matcher;
7494         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
7495                 key->domain ? "FDB" : "NIC", key->table_id,
7496                 cache_matcher->priority,
7497                 key->direction ? "tx" : "rx", (void *)cache_matcher,
7498                 rte_atomic32_read(&cache_matcher->refcnt));
7499         return 0;
7500 }
7501
7502 /**
7503  * Find existing tag resource or create and register a new one.
7504  *
7505  * @param dev[in, out]
7506  *   Pointer to rte_eth_dev structure.
7507  * @param[in, out] tag_be24
7508  *   Tag value in big endian then R-shift 8.
7509  * @parm[in, out] dev_flow
7510  *   Pointer to the dev_flow.
7511  * @param[out] error
7512  *   pointer to error structure.
7513  *
7514  * @return
7515  *   0 on success otherwise -errno and errno is set.
7516  */
7517 static int
7518 flow_dv_tag_resource_register
7519                         (struct rte_eth_dev *dev,
7520                          uint32_t tag_be24,
7521                          struct mlx5_flow *dev_flow,
7522                          struct rte_flow_error *error)
7523 {
7524         struct mlx5_priv *priv = dev->data->dev_private;
7525         struct mlx5_dev_ctx_shared *sh = priv->sh;
7526         struct mlx5_flow_dv_tag_resource *cache_resource;
7527         struct mlx5_hlist_entry *entry;
7528         int ret;
7529
7530         /* Lookup a matching resource from cache. */
7531         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
7532         if (entry) {
7533                 cache_resource = container_of
7534                         (entry, struct mlx5_flow_dv_tag_resource, entry);
7535                 rte_atomic32_inc(&cache_resource->refcnt);
7536                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
7537                 dev_flow->dv.tag_resource = cache_resource;
7538                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
7539                         (void *)cache_resource,
7540                         rte_atomic32_read(&cache_resource->refcnt));
7541                 return 0;
7542         }
7543         /* Register new resource. */
7544         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
7545                                        &dev_flow->handle->dvh.rix_tag);
7546         if (!cache_resource)
7547                 return rte_flow_error_set(error, ENOMEM,
7548                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7549                                           "cannot allocate resource memory");
7550         cache_resource->entry.key = (uint64_t)tag_be24;
7551         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
7552                                                   &cache_resource->action);
7553         if (ret) {
7554                 rte_free(cache_resource);
7555                 return rte_flow_error_set(error, ENOMEM,
7556                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7557                                           NULL, "cannot create action");
7558         }
7559         rte_atomic32_init(&cache_resource->refcnt);
7560         rte_atomic32_inc(&cache_resource->refcnt);
7561         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
7562                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
7563                 rte_free(cache_resource);
7564                 return rte_flow_error_set(error, EEXIST,
7565                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7566                                           NULL, "cannot insert tag");
7567         }
7568         dev_flow->dv.tag_resource = cache_resource;
7569         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
7570                 (void *)cache_resource,
7571                 rte_atomic32_read(&cache_resource->refcnt));
7572         return 0;
7573 }
7574
7575 /**
7576  * Release the tag.
7577  *
7578  * @param dev
7579  *   Pointer to Ethernet device.
7580  * @param tag_idx
7581  *   Tag index.
7582  *
7583  * @return
7584  *   1 while a reference on it exists, 0 when freed.
7585  */
7586 static int
7587 flow_dv_tag_release(struct rte_eth_dev *dev,
7588                     uint32_t tag_idx)
7589 {
7590         struct mlx5_priv *priv = dev->data->dev_private;
7591         struct mlx5_dev_ctx_shared *sh = priv->sh;
7592         struct mlx5_flow_dv_tag_resource *tag;
7593
7594         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7595         if (!tag)
7596                 return 0;
7597         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
7598                 dev->data->port_id, (void *)tag,
7599                 rte_atomic32_read(&tag->refcnt));
7600         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
7601                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
7602                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
7603                 DRV_LOG(DEBUG, "port %u tag %p: removed",
7604                         dev->data->port_id, (void *)tag);
7605                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7606                 return 0;
7607         }
7608         return 1;
7609 }
7610
7611 /**
7612  * Translate port ID action to vport.
7613  *
7614  * @param[in] dev
7615  *   Pointer to rte_eth_dev structure.
7616  * @param[in] action
7617  *   Pointer to the port ID action.
7618  * @param[out] dst_port_id
7619  *   The target port ID.
7620  * @param[out] error
7621  *   Pointer to the error structure.
7622  *
7623  * @return
7624  *   0 on success, a negative errno value otherwise and rte_errno is set.
7625  */
7626 static int
7627 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
7628                                  const struct rte_flow_action *action,
7629                                  uint32_t *dst_port_id,
7630                                  struct rte_flow_error *error)
7631 {
7632         uint32_t port;
7633         struct mlx5_priv *priv;
7634         const struct rte_flow_action_port_id *conf =
7635                         (const struct rte_flow_action_port_id *)action->conf;
7636
7637         port = conf->original ? dev->data->port_id : conf->id;
7638         priv = mlx5_port_to_eswitch_info(port, false);
7639         if (!priv)
7640                 return rte_flow_error_set(error, -rte_errno,
7641                                           RTE_FLOW_ERROR_TYPE_ACTION,
7642                                           NULL,
7643                                           "No eswitch info was found for port");
7644 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
7645         /*
7646          * This parameter is transferred to
7647          * mlx5dv_dr_action_create_dest_ib_port().
7648          */
7649         *dst_port_id = priv->dev_port;
7650 #else
7651         /*
7652          * Legacy mode, no LAG configurations is supported.
7653          * This parameter is transferred to
7654          * mlx5dv_dr_action_create_dest_vport().
7655          */
7656         *dst_port_id = priv->vport_id;
7657 #endif
7658         return 0;
7659 }
7660
7661 /**
7662  * Create a counter with aging configuration.
7663  *
7664  * @param[in] dev
7665  *   Pointer to rte_eth_dev structure.
7666  * @param[out] count
7667  *   Pointer to the counter action configuration.
7668  * @param[in] age
7669  *   Pointer to the aging action configuration.
7670  *
7671  * @return
7672  *   Index to flow counter on success, 0 otherwise.
7673  */
7674 static uint32_t
7675 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
7676                                 struct mlx5_flow *dev_flow,
7677                                 const struct rte_flow_action_count *count,
7678                                 const struct rte_flow_action_age *age)
7679 {
7680         uint32_t counter;
7681         struct mlx5_age_param *age_param;
7682
7683         counter = flow_dv_counter_alloc(dev,
7684                                 count ? count->shared : 0,
7685                                 count ? count->id : 0,
7686                                 dev_flow->dv.group, !!age);
7687         if (!counter || age == NULL)
7688                 return counter;
7689         age_param  = flow_dv_counter_idx_get_age(dev, counter);
7690         /*
7691          * The counter age accuracy may have a bit delay. Have 3/4
7692          * second bias on the timeount in order to let it age in time.
7693          */
7694         age_param->context = age->context ? age->context :
7695                 (void *)(uintptr_t)(dev_flow->flow_idx);
7696         /*
7697          * The counter age accuracy may have a bit delay. Have 3/4
7698          * second bias on the timeount in order to let it age in time.
7699          */
7700         age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
7701         /* Set expire time in unit of 0.1 sec. */
7702         age_param->port_id = dev->data->port_id;
7703         age_param->expire = age_param->timeout +
7704                         rte_rdtsc() / (rte_get_tsc_hz() / 10);
7705         rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
7706         return counter;
7707 }
7708 /**
7709  * Add Tx queue matcher
7710  *
7711  * @param[in] dev
7712  *   Pointer to the dev struct.
7713  * @param[in, out] matcher
7714  *   Flow matcher.
7715  * @param[in, out] key
7716  *   Flow matcher value.
7717  * @param[in] item
7718  *   Flow pattern to translate.
7719  * @param[in] inner
7720  *   Item is inner pattern.
7721  */
7722 static void
7723 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
7724                                 void *matcher, void *key,
7725                                 const struct rte_flow_item *item)
7726 {
7727         const struct mlx5_rte_flow_item_tx_queue *queue_m;
7728         const struct mlx5_rte_flow_item_tx_queue *queue_v;
7729         void *misc_m =
7730                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7731         void *misc_v =
7732                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7733         struct mlx5_txq_ctrl *txq;
7734         uint32_t queue;
7735
7736
7737         queue_m = (const void *)item->mask;
7738         if (!queue_m)
7739                 return;
7740         queue_v = (const void *)item->spec;
7741         if (!queue_v)
7742                 return;
7743         txq = mlx5_txq_get(dev, queue_v->queue);
7744         if (!txq)
7745                 return;
7746         queue = txq->obj->sq->id;
7747         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
7748         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
7749                  queue & queue_m->queue);
7750         mlx5_txq_release(dev, queue_v->queue);
7751 }
7752
7753 /**
7754  * Set the hash fields according to the @p flow information.
7755  *
7756  * @param[in] dev_flow
7757  *   Pointer to the mlx5_flow.
7758  * @param[in] rss_desc
7759  *   Pointer to the mlx5_flow_rss_desc.
7760  */
7761 static void
7762 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
7763                        struct mlx5_flow_rss_desc *rss_desc)
7764 {
7765         uint64_t items = dev_flow->handle->layers;
7766         int rss_inner = 0;
7767         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
7768
7769         dev_flow->hash_fields = 0;
7770 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
7771         if (rss_desc->level >= 2) {
7772                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
7773                 rss_inner = 1;
7774         }
7775 #endif
7776         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
7777             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
7778                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
7779                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7780                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
7781                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7782                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
7783                         else
7784                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
7785                 }
7786         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
7787                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
7788                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
7789                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7790                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
7791                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7792                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
7793                         else
7794                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
7795                 }
7796         }
7797         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
7798             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
7799                 if (rss_types & ETH_RSS_UDP) {
7800                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7801                                 dev_flow->hash_fields |=
7802                                                 IBV_RX_HASH_SRC_PORT_UDP;
7803                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7804                                 dev_flow->hash_fields |=
7805                                                 IBV_RX_HASH_DST_PORT_UDP;
7806                         else
7807                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
7808                 }
7809         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
7810                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
7811                 if (rss_types & ETH_RSS_TCP) {
7812                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7813                                 dev_flow->hash_fields |=
7814                                                 IBV_RX_HASH_SRC_PORT_TCP;
7815                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7816                                 dev_flow->hash_fields |=
7817                                                 IBV_RX_HASH_DST_PORT_TCP;
7818                         else
7819                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
7820                 }
7821         }
7822 }
7823
7824 /**
7825  * Fill the flow with DV spec, lock free
7826  * (mutex should be acquired by caller).
7827  *
7828  * @param[in] dev
7829  *   Pointer to rte_eth_dev structure.
7830  * @param[in, out] dev_flow
7831  *   Pointer to the sub flow.
7832  * @param[in] attr
7833  *   Pointer to the flow attributes.
7834  * @param[in] items
7835  *   Pointer to the list of items.
7836  * @param[in] actions
7837  *   Pointer to the list of actions.
7838  * @param[out] error
7839  *   Pointer to the error structure.
7840  *
7841  * @return
7842  *   0 on success, a negative errno value otherwise and rte_errno is set.
7843  */
7844 static int
7845 __flow_dv_translate(struct rte_eth_dev *dev,
7846                     struct mlx5_flow *dev_flow,
7847                     const struct rte_flow_attr *attr,
7848                     const struct rte_flow_item items[],
7849                     const struct rte_flow_action actions[],
7850                     struct rte_flow_error *error)
7851 {
7852         struct mlx5_priv *priv = dev->data->dev_private;
7853         struct mlx5_dev_config *dev_conf = &priv->config;
7854         struct rte_flow *flow = dev_flow->flow;
7855         struct mlx5_flow_handle *handle = dev_flow->handle;
7856         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
7857                                               priv->rss_desc)
7858                                               [!!priv->flow_nested_idx];
7859         uint64_t item_flags = 0;
7860         uint64_t last_item = 0;
7861         uint64_t action_flags = 0;
7862         uint64_t priority = attr->priority;
7863         struct mlx5_flow_dv_matcher matcher = {
7864                 .mask = {
7865                         .size = sizeof(matcher.mask.buf),
7866                 },
7867         };
7868         int actions_n = 0;
7869         bool actions_end = false;
7870         union {
7871                 struct mlx5_flow_dv_modify_hdr_resource res;
7872                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
7873                             sizeof(struct mlx5_modification_cmd) *
7874                             (MLX5_MAX_MODIFY_NUM + 1)];
7875         } mhdr_dummy;
7876         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
7877         const struct rte_flow_action_count *count = NULL;
7878         const struct rte_flow_action_age *age = NULL;
7879         union flow_dv_attr flow_attr = { .attr = 0 };
7880         uint32_t tag_be;
7881         union mlx5_flow_tbl_key tbl_key;
7882         uint32_t modify_action_position = UINT32_MAX;
7883         void *match_mask = matcher.mask.buf;
7884         void *match_value = dev_flow->dv.value.buf;
7885         uint8_t next_protocol = 0xff;
7886         struct rte_vlan_hdr vlan = { 0 };
7887         uint32_t table;
7888         int ret = 0;
7889
7890         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
7891                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
7892         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
7893                                        !!priv->fdb_def_rule, &table, error);
7894         if (ret)
7895                 return ret;
7896         dev_flow->dv.group = table;
7897         if (attr->transfer)
7898                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
7899         if (priority == MLX5_FLOW_PRIO_RSVD)
7900                 priority = dev_conf->flow_prio - 1;
7901         /* number of actions must be set to 0 in case of dirty stack. */
7902         mhdr_res->actions_num = 0;
7903         for (; !actions_end ; actions++) {
7904                 const struct rte_flow_action_queue *queue;
7905                 const struct rte_flow_action_rss *rss;
7906                 const struct rte_flow_action *action = actions;
7907                 const uint8_t *rss_key;
7908                 const struct rte_flow_action_jump *jump_data;
7909                 const struct rte_flow_action_meter *mtr;
7910                 struct mlx5_flow_tbl_resource *tbl;
7911                 uint32_t port_id = 0;
7912                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
7913                 int action_type = actions->type;
7914                 const struct rte_flow_action *found_action = NULL;
7915                 struct mlx5_flow_meter *fm = NULL;
7916
7917                 if (!mlx5_flow_os_action_supported(action_type))
7918                         return rte_flow_error_set(error, ENOTSUP,
7919                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7920                                                   actions,
7921                                                   "action not supported");
7922                 switch (action_type) {
7923                 case RTE_FLOW_ACTION_TYPE_VOID:
7924                         break;
7925                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7926                         if (flow_dv_translate_action_port_id(dev, action,
7927                                                              &port_id, error))
7928                                 return -rte_errno;
7929                         port_id_resource.port_id = port_id;
7930                         MLX5_ASSERT(!handle->rix_port_id_action);
7931                         if (flow_dv_port_id_action_resource_register
7932                             (dev, &port_id_resource, dev_flow, error))
7933                                 return -rte_errno;
7934                         dev_flow->dv.actions[actions_n++] =
7935                                         dev_flow->dv.port_id_action->action;
7936                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7937                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
7938                         break;
7939                 case RTE_FLOW_ACTION_TYPE_FLAG:
7940                         action_flags |= MLX5_FLOW_ACTION_FLAG;
7941                         dev_flow->handle->mark = 1;
7942                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7943                                 struct rte_flow_action_mark mark = {
7944                                         .id = MLX5_FLOW_MARK_DEFAULT,
7945                                 };
7946
7947                                 if (flow_dv_convert_action_mark(dev, &mark,
7948                                                                 mhdr_res,
7949                                                                 error))
7950                                         return -rte_errno;
7951                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
7952                                 break;
7953                         }
7954                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
7955                         /*
7956                          * Only one FLAG or MARK is supported per device flow
7957                          * right now. So the pointer to the tag resource must be
7958                          * zero before the register process.
7959                          */
7960                         MLX5_ASSERT(!handle->dvh.rix_tag);
7961                         if (flow_dv_tag_resource_register(dev, tag_be,
7962                                                           dev_flow, error))
7963                                 return -rte_errno;
7964                         MLX5_ASSERT(dev_flow->dv.tag_resource);
7965                         dev_flow->dv.actions[actions_n++] =
7966                                         dev_flow->dv.tag_resource->action;
7967                         break;
7968                 case RTE_FLOW_ACTION_TYPE_MARK:
7969                         action_flags |= MLX5_FLOW_ACTION_MARK;
7970                         dev_flow->handle->mark = 1;
7971                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7972                                 const struct rte_flow_action_mark *mark =
7973                                         (const struct rte_flow_action_mark *)
7974                                                 actions->conf;
7975
7976                                 if (flow_dv_convert_action_mark(dev, mark,
7977                                                                 mhdr_res,
7978                                                                 error))
7979                                         return -rte_errno;
7980                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
7981                                 break;
7982                         }
7983                         /* Fall-through */
7984                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7985                         /* Legacy (non-extensive) MARK action. */
7986                         tag_be = mlx5_flow_mark_set
7987                               (((const struct rte_flow_action_mark *)
7988                                (actions->conf))->id);
7989                         MLX5_ASSERT(!handle->dvh.rix_tag);
7990                         if (flow_dv_tag_resource_register(dev, tag_be,
7991                                                           dev_flow, error))
7992                                 return -rte_errno;
7993                         MLX5_ASSERT(dev_flow->dv.tag_resource);
7994                         dev_flow->dv.actions[actions_n++] =
7995                                         dev_flow->dv.tag_resource->action;
7996                         break;
7997                 case RTE_FLOW_ACTION_TYPE_SET_META:
7998                         if (flow_dv_convert_action_set_meta
7999                                 (dev, mhdr_res, attr,
8000                                  (const struct rte_flow_action_set_meta *)
8001                                   actions->conf, error))
8002                                 return -rte_errno;
8003                         action_flags |= MLX5_FLOW_ACTION_SET_META;
8004                         break;
8005                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
8006                         if (flow_dv_convert_action_set_tag
8007                                 (dev, mhdr_res,
8008                                  (const struct rte_flow_action_set_tag *)
8009                                   actions->conf, error))
8010                                 return -rte_errno;
8011                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8012                         break;
8013                 case RTE_FLOW_ACTION_TYPE_DROP:
8014                         action_flags |= MLX5_FLOW_ACTION_DROP;
8015                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
8016                         break;
8017                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8018                         queue = actions->conf;
8019                         rss_desc->queue_num = 1;
8020                         rss_desc->queue[0] = queue->index;
8021                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8022                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8023                         break;
8024                 case RTE_FLOW_ACTION_TYPE_RSS:
8025                         rss = actions->conf;
8026                         memcpy(rss_desc->queue, rss->queue,
8027                                rss->queue_num * sizeof(uint16_t));
8028                         rss_desc->queue_num = rss->queue_num;
8029                         /* NULL RSS key indicates default RSS key. */
8030                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
8031                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
8032                         /*
8033                          * rss->level and rss.types should be set in advance
8034                          * when expanding items for RSS.
8035                          */
8036                         action_flags |= MLX5_FLOW_ACTION_RSS;
8037                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8038                         break;
8039                 case RTE_FLOW_ACTION_TYPE_AGE:
8040                 case RTE_FLOW_ACTION_TYPE_COUNT:
8041                         if (!dev_conf->devx) {
8042                                 return rte_flow_error_set
8043                                               (error, ENOTSUP,
8044                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8045                                                NULL,
8046                                                "count action not supported");
8047                         }
8048                         /* Save information first, will apply later. */
8049                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
8050                                 count = action->conf;
8051                         else
8052                                 age = action->conf;
8053                         action_flags |= MLX5_FLOW_ACTION_COUNT;
8054                         break;
8055                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
8056                         dev_flow->dv.actions[actions_n++] =
8057                                                 priv->sh->pop_vlan_action;
8058                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
8059                         break;
8060                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
8061                         if (!(action_flags &
8062                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
8063                                 flow_dev_get_vlan_info_from_items(items, &vlan);
8064                         vlan.eth_proto = rte_be_to_cpu_16
8065                              ((((const struct rte_flow_action_of_push_vlan *)
8066                                                    actions->conf)->ethertype));
8067                         found_action = mlx5_flow_find_action
8068                                         (actions + 1,
8069                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
8070                         if (found_action)
8071                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8072                         found_action = mlx5_flow_find_action
8073                                         (actions + 1,
8074                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
8075                         if (found_action)
8076                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8077                         if (flow_dv_create_action_push_vlan
8078                                             (dev, attr, &vlan, dev_flow, error))
8079                                 return -rte_errno;
8080                         dev_flow->dv.actions[actions_n++] =
8081                                         dev_flow->dv.push_vlan_res->action;
8082                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
8083                         break;
8084                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
8085                         /* of_vlan_push action handled this action */
8086                         MLX5_ASSERT(action_flags &
8087                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
8088                         break;
8089                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
8090                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8091                                 break;
8092                         flow_dev_get_vlan_info_from_items(items, &vlan);
8093                         mlx5_update_vlan_vid_pcp(actions, &vlan);
8094                         /* If no VLAN push - this is a modify header action */
8095                         if (flow_dv_convert_action_modify_vlan_vid
8096                                                 (mhdr_res, actions, error))
8097                                 return -rte_errno;
8098                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
8099                         break;
8100                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
8101                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
8102                         if (flow_dv_create_action_l2_encap(dev, actions,
8103                                                            dev_flow,
8104                                                            attr->transfer,
8105                                                            error))
8106                                 return -rte_errno;
8107                         dev_flow->dv.actions[actions_n++] =
8108                                         dev_flow->dv.encap_decap->action;
8109                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8110                         break;
8111                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
8112                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
8113                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
8114                                                            attr->transfer,
8115                                                            error))
8116                                 return -rte_errno;
8117                         dev_flow->dv.actions[actions_n++] =
8118                                         dev_flow->dv.encap_decap->action;
8119                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8120                         break;
8121                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8122                         /* Handle encap with preceding decap. */
8123                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
8124                                 if (flow_dv_create_action_raw_encap
8125                                         (dev, actions, dev_flow, attr, error))
8126                                         return -rte_errno;
8127                                 dev_flow->dv.actions[actions_n++] =
8128                                         dev_flow->dv.encap_decap->action;
8129                         } else {
8130                                 /* Handle encap without preceding decap. */
8131                                 if (flow_dv_create_action_l2_encap
8132                                     (dev, actions, dev_flow, attr->transfer,
8133                                      error))
8134                                         return -rte_errno;
8135                                 dev_flow->dv.actions[actions_n++] =
8136                                         dev_flow->dv.encap_decap->action;
8137                         }
8138                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8139                         break;
8140                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
8141                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
8142                                 ;
8143                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
8144                                 if (flow_dv_create_action_l2_decap
8145                                     (dev, dev_flow, attr->transfer, error))
8146                                         return -rte_errno;
8147                                 dev_flow->dv.actions[actions_n++] =
8148                                         dev_flow->dv.encap_decap->action;
8149                         }
8150                         /* If decap is followed by encap, handle it at encap. */
8151                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8152                         break;
8153                 case RTE_FLOW_ACTION_TYPE_JUMP:
8154                         jump_data = action->conf;
8155                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
8156                                                        jump_data->group,
8157                                                        !!priv->fdb_def_rule,
8158                                                        &table, error);
8159                         if (ret)
8160                                 return ret;
8161                         tbl = flow_dv_tbl_resource_get(dev, table,
8162                                                        attr->egress,
8163                                                        attr->transfer, error);
8164                         if (!tbl)
8165                                 return rte_flow_error_set
8166                                                 (error, errno,
8167                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8168                                                  NULL,
8169                                                  "cannot create jump action.");
8170                         if (flow_dv_jump_tbl_resource_register
8171                             (dev, tbl, dev_flow, error)) {
8172                                 flow_dv_tbl_resource_release(dev, tbl);
8173                                 return rte_flow_error_set
8174                                                 (error, errno,
8175                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8176                                                  NULL,
8177                                                  "cannot create jump action.");
8178                         }
8179                         dev_flow->dv.actions[actions_n++] =
8180                                         dev_flow->dv.jump->action;
8181                         action_flags |= MLX5_FLOW_ACTION_JUMP;
8182                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
8183                         break;
8184                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
8185                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
8186                         if (flow_dv_convert_action_modify_mac
8187                                         (mhdr_res, actions, error))
8188                                 return -rte_errno;
8189                         action_flags |= actions->type ==
8190                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
8191                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
8192                                         MLX5_FLOW_ACTION_SET_MAC_DST;
8193                         break;
8194                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
8195                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
8196                         if (flow_dv_convert_action_modify_ipv4
8197                                         (mhdr_res, actions, error))
8198                                 return -rte_errno;
8199                         action_flags |= actions->type ==
8200                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
8201                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
8202                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
8203                         break;
8204                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
8205                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
8206                         if (flow_dv_convert_action_modify_ipv6
8207                                         (mhdr_res, actions, error))
8208                                 return -rte_errno;
8209                         action_flags |= actions->type ==
8210                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
8211                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
8212                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
8213                         break;
8214                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
8215                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
8216                         if (flow_dv_convert_action_modify_tp
8217                                         (mhdr_res, actions, items,
8218                                          &flow_attr, dev_flow, !!(action_flags &
8219                                          MLX5_FLOW_ACTION_DECAP), error))
8220                                 return -rte_errno;
8221                         action_flags |= actions->type ==
8222                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
8223                                         MLX5_FLOW_ACTION_SET_TP_SRC :
8224                                         MLX5_FLOW_ACTION_SET_TP_DST;
8225                         break;
8226                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
8227                         if (flow_dv_convert_action_modify_dec_ttl
8228                                         (mhdr_res, items, &flow_attr, dev_flow,
8229                                          !!(action_flags &
8230                                          MLX5_FLOW_ACTION_DECAP), error))
8231                                 return -rte_errno;
8232                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
8233                         break;
8234                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
8235                         if (flow_dv_convert_action_modify_ttl
8236                                         (mhdr_res, actions, items, &flow_attr,
8237                                          dev_flow, !!(action_flags &
8238                                          MLX5_FLOW_ACTION_DECAP), error))
8239                                 return -rte_errno;
8240                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
8241                         break;
8242                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
8243                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
8244                         if (flow_dv_convert_action_modify_tcp_seq
8245                                         (mhdr_res, actions, error))
8246                                 return -rte_errno;
8247                         action_flags |= actions->type ==
8248                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
8249                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
8250                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
8251                         break;
8252
8253                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
8254                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
8255                         if (flow_dv_convert_action_modify_tcp_ack
8256                                         (mhdr_res, actions, error))
8257                                 return -rte_errno;
8258                         action_flags |= actions->type ==
8259                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
8260                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
8261                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
8262                         break;
8263                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
8264                         if (flow_dv_convert_action_set_reg
8265                                         (mhdr_res, actions, error))
8266                                 return -rte_errno;
8267                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8268                         break;
8269                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
8270                         if (flow_dv_convert_action_copy_mreg
8271                                         (dev, mhdr_res, actions, error))
8272                                 return -rte_errno;
8273                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8274                         break;
8275                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
8276                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
8277                         dev_flow->handle->fate_action =
8278                                         MLX5_FLOW_FATE_DEFAULT_MISS;
8279                         break;
8280                 case RTE_FLOW_ACTION_TYPE_METER:
8281                         mtr = actions->conf;
8282                         if (!flow->meter) {
8283                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
8284                                                             attr, error);
8285                                 if (!fm)
8286                                         return rte_flow_error_set(error,
8287                                                 rte_errno,
8288                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8289                                                 NULL,
8290                                                 "meter not found "
8291                                                 "or invalid parameters");
8292                                 flow->meter = fm->idx;
8293                         }
8294                         /* Set the meter action. */
8295                         if (!fm) {
8296                                 fm = mlx5_ipool_get(priv->sh->ipool
8297                                                 [MLX5_IPOOL_MTR], flow->meter);
8298                                 if (!fm)
8299                                         return rte_flow_error_set(error,
8300                                                 rte_errno,
8301                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8302                                                 NULL,
8303                                                 "meter not found "
8304                                                 "or invalid parameters");
8305                         }
8306                         dev_flow->dv.actions[actions_n++] =
8307                                 fm->mfts->meter_action;
8308                         action_flags |= MLX5_FLOW_ACTION_METER;
8309                         break;
8310                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
8311                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
8312                                                               actions, error))
8313                                 return -rte_errno;
8314                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
8315                         break;
8316                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
8317                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
8318                                                               actions, error))
8319                                 return -rte_errno;
8320                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
8321                         break;
8322                 case RTE_FLOW_ACTION_TYPE_END:
8323                         actions_end = true;
8324                         if (mhdr_res->actions_num) {
8325                                 /* create modify action if needed. */
8326                                 if (flow_dv_modify_hdr_resource_register
8327                                         (dev, mhdr_res, dev_flow, error))
8328                                         return -rte_errno;
8329                                 dev_flow->dv.actions[modify_action_position] =
8330                                         handle->dvh.modify_hdr->action;
8331                         }
8332                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
8333                                 flow->counter =
8334                                         flow_dv_translate_create_counter(dev,
8335                                                 dev_flow, count, age);
8336
8337                                 if (!flow->counter)
8338                                         return rte_flow_error_set
8339                                                 (error, rte_errno,
8340                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8341                                                 NULL,
8342                                                 "cannot create counter"
8343                                                 " object.");
8344                                 dev_flow->dv.actions[actions_n++] =
8345                                           (flow_dv_counter_get_by_idx(dev,
8346                                           flow->counter, NULL))->action;
8347                         }
8348                         break;
8349                 default:
8350                         break;
8351                 }
8352                 if (mhdr_res->actions_num &&
8353                     modify_action_position == UINT32_MAX)
8354                         modify_action_position = actions_n++;
8355         }
8356         dev_flow->dv.actions_n = actions_n;
8357         dev_flow->act_flags = action_flags;
8358         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
8359                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
8360                 int item_type = items->type;
8361
8362                 if (!mlx5_flow_os_item_supported(item_type))
8363                         return rte_flow_error_set(error, ENOTSUP,
8364                                                   RTE_FLOW_ERROR_TYPE_ITEM,
8365                                                   NULL, "item not supported");
8366                 switch (item_type) {
8367                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
8368                         flow_dv_translate_item_port_id(dev, match_mask,
8369                                                        match_value, items);
8370                         last_item = MLX5_FLOW_ITEM_PORT_ID;
8371                         break;
8372                 case RTE_FLOW_ITEM_TYPE_ETH:
8373                         flow_dv_translate_item_eth(match_mask, match_value,
8374                                                    items, tunnel,
8375                                                    dev_flow->dv.group);
8376                         matcher.priority = action_flags &
8377                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
8378                                         !dev_flow->external ?
8379                                         MLX5_PRIORITY_MAP_L3 :
8380                                         MLX5_PRIORITY_MAP_L2;
8381                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
8382                                              MLX5_FLOW_LAYER_OUTER_L2;
8383                         break;
8384                 case RTE_FLOW_ITEM_TYPE_VLAN:
8385                         flow_dv_translate_item_vlan(dev_flow,
8386                                                     match_mask, match_value,
8387                                                     items, tunnel,
8388                                                     dev_flow->dv.group);
8389                         matcher.priority = MLX5_PRIORITY_MAP_L2;
8390                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
8391                                               MLX5_FLOW_LAYER_INNER_VLAN) :
8392                                              (MLX5_FLOW_LAYER_OUTER_L2 |
8393                                               MLX5_FLOW_LAYER_OUTER_VLAN);
8394                         break;
8395                 case RTE_FLOW_ITEM_TYPE_IPV4:
8396                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8397                                                   &item_flags, &tunnel);
8398                         flow_dv_translate_item_ipv4(match_mask, match_value,
8399                                                     items, item_flags, tunnel,
8400                                                     dev_flow->dv.group);
8401                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8402                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
8403                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
8404                         if (items->mask != NULL &&
8405                             ((const struct rte_flow_item_ipv4 *)
8406                              items->mask)->hdr.next_proto_id) {
8407                                 next_protocol =
8408                                         ((const struct rte_flow_item_ipv4 *)
8409                                          (items->spec))->hdr.next_proto_id;
8410                                 next_protocol &=
8411                                         ((const struct rte_flow_item_ipv4 *)
8412                                          (items->mask))->hdr.next_proto_id;
8413                         } else {
8414                                 /* Reset for inner layer. */
8415                                 next_protocol = 0xff;
8416                         }
8417                         break;
8418                 case RTE_FLOW_ITEM_TYPE_IPV6:
8419                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8420                                                   &item_flags, &tunnel);
8421                         flow_dv_translate_item_ipv6(match_mask, match_value,
8422                                                     items, item_flags, tunnel,
8423                                                     dev_flow->dv.group);
8424                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8425                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
8426                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
8427                         if (items->mask != NULL &&
8428                             ((const struct rte_flow_item_ipv6 *)
8429                              items->mask)->hdr.proto) {
8430                                 next_protocol =
8431                                         ((const struct rte_flow_item_ipv6 *)
8432                                          items->spec)->hdr.proto;
8433                                 next_protocol &=
8434                                         ((const struct rte_flow_item_ipv6 *)
8435                                          items->mask)->hdr.proto;
8436                         } else {
8437                                 /* Reset for inner layer. */
8438                                 next_protocol = 0xff;
8439                         }
8440                         break;
8441                 case RTE_FLOW_ITEM_TYPE_TCP:
8442                         flow_dv_translate_item_tcp(match_mask, match_value,
8443                                                    items, tunnel);
8444                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8445                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
8446                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
8447                         break;
8448                 case RTE_FLOW_ITEM_TYPE_UDP:
8449                         flow_dv_translate_item_udp(match_mask, match_value,
8450                                                    items, tunnel);
8451                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8452                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
8453                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
8454                         break;
8455                 case RTE_FLOW_ITEM_TYPE_GRE:
8456                         flow_dv_translate_item_gre(match_mask, match_value,
8457                                                    items, tunnel);
8458                         matcher.priority = rss_desc->level >= 2 ?
8459                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8460                         last_item = MLX5_FLOW_LAYER_GRE;
8461                         break;
8462                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
8463                         flow_dv_translate_item_gre_key(match_mask,
8464                                                        match_value, items);
8465                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
8466                         break;
8467                 case RTE_FLOW_ITEM_TYPE_NVGRE:
8468                         flow_dv_translate_item_nvgre(match_mask, match_value,
8469                                                      items, tunnel);
8470                         matcher.priority = rss_desc->level >= 2 ?
8471                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8472                         last_item = MLX5_FLOW_LAYER_GRE;
8473                         break;
8474                 case RTE_FLOW_ITEM_TYPE_VXLAN:
8475                         flow_dv_translate_item_vxlan(match_mask, match_value,
8476                                                      items, tunnel);
8477                         matcher.priority = rss_desc->level >= 2 ?
8478                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8479                         last_item = MLX5_FLOW_LAYER_VXLAN;
8480                         break;
8481                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
8482                         flow_dv_translate_item_vxlan_gpe(match_mask,
8483                                                          match_value, items,
8484                                                          tunnel);
8485                         matcher.priority = rss_desc->level >= 2 ?
8486                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8487                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
8488                         break;
8489                 case RTE_FLOW_ITEM_TYPE_GENEVE:
8490                         flow_dv_translate_item_geneve(match_mask, match_value,
8491                                                       items, tunnel);
8492                         matcher.priority = rss_desc->level >= 2 ?
8493                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8494                         last_item = MLX5_FLOW_LAYER_GENEVE;
8495                         break;
8496                 case RTE_FLOW_ITEM_TYPE_MPLS:
8497                         flow_dv_translate_item_mpls(match_mask, match_value,
8498                                                     items, last_item, tunnel);
8499                         matcher.priority = rss_desc->level >= 2 ?
8500                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8501                         last_item = MLX5_FLOW_LAYER_MPLS;
8502                         break;
8503                 case RTE_FLOW_ITEM_TYPE_MARK:
8504                         flow_dv_translate_item_mark(dev, match_mask,
8505                                                     match_value, items);
8506                         last_item = MLX5_FLOW_ITEM_MARK;
8507                         break;
8508                 case RTE_FLOW_ITEM_TYPE_META:
8509                         flow_dv_translate_item_meta(dev, match_mask,
8510                                                     match_value, attr, items);
8511                         last_item = MLX5_FLOW_ITEM_METADATA;
8512                         break;
8513                 case RTE_FLOW_ITEM_TYPE_ICMP:
8514                         flow_dv_translate_item_icmp(match_mask, match_value,
8515                                                     items, tunnel);
8516                         last_item = MLX5_FLOW_LAYER_ICMP;
8517                         break;
8518                 case RTE_FLOW_ITEM_TYPE_ICMP6:
8519                         flow_dv_translate_item_icmp6(match_mask, match_value,
8520                                                       items, tunnel);
8521                         last_item = MLX5_FLOW_LAYER_ICMP6;
8522                         break;
8523                 case RTE_FLOW_ITEM_TYPE_TAG:
8524                         flow_dv_translate_item_tag(dev, match_mask,
8525                                                    match_value, items);
8526                         last_item = MLX5_FLOW_ITEM_TAG;
8527                         break;
8528                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
8529                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
8530                                                         match_value, items);
8531                         last_item = MLX5_FLOW_ITEM_TAG;
8532                         break;
8533                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
8534                         flow_dv_translate_item_tx_queue(dev, match_mask,
8535                                                         match_value,
8536                                                         items);
8537                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
8538                         break;
8539                 case RTE_FLOW_ITEM_TYPE_GTP:
8540                         flow_dv_translate_item_gtp(match_mask, match_value,
8541                                                    items, tunnel);
8542                         matcher.priority = rss_desc->level >= 2 ?
8543                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8544                         last_item = MLX5_FLOW_LAYER_GTP;
8545                         break;
8546                 default:
8547                         break;
8548                 }
8549                 item_flags |= last_item;
8550         }
8551         /*
8552          * When E-Switch mode is enabled, we have two cases where we need to
8553          * set the source port manually.
8554          * The first one, is in case of Nic steering rule, and the second is
8555          * E-Switch rule where no port_id item was found. In both cases
8556          * the source port is set according the current port in use.
8557          */
8558         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
8559             (priv->representor || priv->master)) {
8560                 if (flow_dv_translate_item_port_id(dev, match_mask,
8561                                                    match_value, NULL))
8562                         return -rte_errno;
8563         }
8564 #ifdef RTE_LIBRTE_MLX5_DEBUG
8565         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
8566                                               dev_flow->dv.value.buf));
8567 #endif
8568         /*
8569          * Layers may be already initialized from prefix flow if this dev_flow
8570          * is the suffix flow.
8571          */
8572         handle->layers |= item_flags;
8573         if (action_flags & MLX5_FLOW_ACTION_RSS)
8574                 flow_dv_hashfields_set(dev_flow, rss_desc);
8575         /* Register matcher. */
8576         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
8577                                     matcher.mask.size);
8578         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
8579                                                      matcher.priority);
8580         /* reserved field no needs to be set to 0 here. */
8581         tbl_key.domain = attr->transfer;
8582         tbl_key.direction = attr->egress;
8583         tbl_key.table_id = dev_flow->dv.group;
8584         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
8585                 return -rte_errno;
8586         return 0;
8587 }
8588
8589 /**
8590  * Apply the flow to the NIC, lock free,
8591  * (mutex should be acquired by caller).
8592  *
8593  * @param[in] dev
8594  *   Pointer to the Ethernet device structure.
8595  * @param[in, out] flow
8596  *   Pointer to flow structure.
8597  * @param[out] error
8598  *   Pointer to error structure.
8599  *
8600  * @return
8601  *   0 on success, a negative errno value otherwise and rte_errno is set.
8602  */
8603 static int
8604 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
8605                 struct rte_flow_error *error)
8606 {
8607         struct mlx5_flow_dv_workspace *dv;
8608         struct mlx5_flow_handle *dh;
8609         struct mlx5_flow_handle_dv *dv_h;
8610         struct mlx5_flow *dev_flow;
8611         struct mlx5_priv *priv = dev->data->dev_private;
8612         uint32_t handle_idx;
8613         int n;
8614         int err;
8615         int idx;
8616
8617         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
8618                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
8619                 dv = &dev_flow->dv;
8620                 dh = dev_flow->handle;
8621                 dv_h = &dh->dvh;
8622                 n = dv->actions_n;
8623                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8624                         if (dv->transfer) {
8625                                 dv->actions[n++] = priv->sh->esw_drop_action;
8626                         } else {
8627                                 struct mlx5_hrxq *drop_hrxq;
8628                                 drop_hrxq = mlx5_hrxq_drop_new(dev);
8629                                 if (!drop_hrxq) {
8630                                         rte_flow_error_set
8631                                                 (error, errno,
8632                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8633                                                  NULL,
8634                                                  "cannot get drop hash queue");
8635                                         goto error;
8636                                 }
8637                                 /*
8638                                  * Drop queues will be released by the specify
8639                                  * mlx5_hrxq_drop_release() function. Assign
8640                                  * the special index to hrxq to mark the queue
8641                                  * has been allocated.
8642                                  */
8643                                 dh->rix_hrxq = UINT32_MAX;
8644                                 dv->actions[n++] = drop_hrxq->action;
8645                         }
8646                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8647                         struct mlx5_hrxq *hrxq;
8648                         uint32_t hrxq_idx;
8649                         struct mlx5_flow_rss_desc *rss_desc =
8650                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
8651                                 [!!priv->flow_nested_idx];
8652
8653                         MLX5_ASSERT(rss_desc->queue_num);
8654                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8655                                                  MLX5_RSS_HASH_KEY_LEN,
8656                                                  dev_flow->hash_fields,
8657                                                  rss_desc->queue,
8658                                                  rss_desc->queue_num);
8659                         if (!hrxq_idx) {
8660                                 hrxq_idx = mlx5_hrxq_new
8661                                                 (dev, rss_desc->key,
8662                                                 MLX5_RSS_HASH_KEY_LEN,
8663                                                 dev_flow->hash_fields,
8664                                                 rss_desc->queue,
8665                                                 rss_desc->queue_num,
8666                                                 !!(dh->layers &
8667                                                 MLX5_FLOW_LAYER_TUNNEL));
8668                         }
8669                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8670                                               hrxq_idx);
8671                         if (!hrxq) {
8672                                 rte_flow_error_set
8673                                         (error, rte_errno,
8674                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8675                                          "cannot get hash queue");
8676                                 goto error;
8677                         }
8678                         dh->rix_hrxq = hrxq_idx;
8679                         dv->actions[n++] = hrxq->action;
8680                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
8681                         if (flow_dv_default_miss_resource_register
8682                                         (dev, error)) {
8683                                 rte_flow_error_set
8684                                         (error, rte_errno,
8685                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8686                                          "cannot create default miss resource");
8687                                 goto error_default_miss;
8688                         }
8689                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
8690                         dv->actions[n++] = priv->sh->default_miss.action;
8691                 }
8692                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
8693                                                (void *)&dv->value, n,
8694                                                dv->actions, &dh->drv_flow);
8695                 if (err) {
8696                         rte_flow_error_set(error, errno,
8697                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8698                                            NULL,
8699                                            "hardware refuses to create flow");
8700                         goto error;
8701                 }
8702                 if (priv->vmwa_context &&
8703                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
8704                         /*
8705                          * The rule contains the VLAN pattern.
8706                          * For VF we are going to create VLAN
8707                          * interface to make hypervisor set correct
8708                          * e-Switch vport context.
8709                          */
8710                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
8711                 }
8712         }
8713         return 0;
8714 error:
8715         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
8716                 flow_dv_default_miss_resource_release(dev);
8717 error_default_miss:
8718         err = rte_errno; /* Save rte_errno before cleanup. */
8719         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
8720                        handle_idx, dh, next) {
8721                 /* hrxq is union, don't clear it if the flag is not set. */
8722                 if (dh->rix_hrxq) {
8723                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8724                                 mlx5_hrxq_drop_release(dev);
8725                                 dh->rix_hrxq = 0;
8726                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8727                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
8728                                 dh->rix_hrxq = 0;
8729                         }
8730                 }
8731                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
8732                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
8733         }
8734         rte_errno = err; /* Restore rte_errno. */
8735         return -rte_errno;
8736 }
8737
8738 /**
8739  * Release the flow matcher.
8740  *
8741  * @param dev
8742  *   Pointer to Ethernet device.
8743  * @param handle
8744  *   Pointer to mlx5_flow_handle.
8745  *
8746  * @return
8747  *   1 while a reference on it exists, 0 when freed.
8748  */
8749 static int
8750 flow_dv_matcher_release(struct rte_eth_dev *dev,
8751                         struct mlx5_flow_handle *handle)
8752 {
8753         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
8754
8755         MLX5_ASSERT(matcher->matcher_object);
8756         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
8757                 dev->data->port_id, (void *)matcher,
8758                 rte_atomic32_read(&matcher->refcnt));
8759         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
8760                 claim_zero(mlx5_flow_os_destroy_flow_matcher
8761                            (matcher->matcher_object));
8762                 LIST_REMOVE(matcher, next);
8763                 /* table ref-- in release interface. */
8764                 flow_dv_tbl_resource_release(dev, matcher->tbl);
8765                 rte_free(matcher);
8766                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
8767                         dev->data->port_id, (void *)matcher);
8768                 return 0;
8769         }
8770         return 1;
8771 }
8772
8773 /**
8774  * Release an encap/decap resource.
8775  *
8776  * @param dev
8777  *   Pointer to Ethernet device.
8778  * @param handle
8779  *   Pointer to mlx5_flow_handle.
8780  *
8781  * @return
8782  *   1 while a reference on it exists, 0 when freed.
8783  */
8784 static int
8785 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
8786                                      struct mlx5_flow_handle *handle)
8787 {
8788         struct mlx5_priv *priv = dev->data->dev_private;
8789         uint32_t idx = handle->dvh.rix_encap_decap;
8790         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
8791
8792         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8793                          idx);
8794         if (!cache_resource)
8795                 return 0;
8796         MLX5_ASSERT(cache_resource->action);
8797         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
8798                 (void *)cache_resource,
8799                 rte_atomic32_read(&cache_resource->refcnt));
8800         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8801                 claim_zero(mlx5_flow_os_destroy_flow_action
8802                                                 (cache_resource->action));
8803                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8804                              &priv->sh->encaps_decaps, idx,
8805                              cache_resource, next);
8806                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
8807                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
8808                         (void *)cache_resource);
8809                 return 0;
8810         }
8811         return 1;
8812 }
8813
8814 /**
8815  * Release an jump to table action resource.
8816  *
8817  * @param dev
8818  *   Pointer to Ethernet device.
8819  * @param handle
8820  *   Pointer to mlx5_flow_handle.
8821  *
8822  * @return
8823  *   1 while a reference on it exists, 0 when freed.
8824  */
8825 static int
8826 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
8827                                   struct mlx5_flow_handle *handle)
8828 {
8829         struct mlx5_priv *priv = dev->data->dev_private;
8830         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
8831         struct mlx5_flow_tbl_data_entry *tbl_data;
8832
8833         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
8834                              handle->rix_jump);
8835         if (!tbl_data)
8836                 return 0;
8837         cache_resource = &tbl_data->jump;
8838         MLX5_ASSERT(cache_resource->action);
8839         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
8840                 (void *)cache_resource,
8841                 rte_atomic32_read(&cache_resource->refcnt));
8842         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8843                 claim_zero(mlx5_flow_os_destroy_flow_action
8844                                                 (cache_resource->action));
8845                 /* jump action memory free is inside the table release. */
8846                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
8847                 DRV_LOG(DEBUG, "jump table resource %p: removed",
8848                         (void *)cache_resource);
8849                 return 0;
8850         }
8851         return 1;
8852 }
8853
8854 /**
8855  * Release a default miss resource.
8856  *
8857  * @param dev
8858  *   Pointer to Ethernet device.
8859  * @return
8860  *   1 while a reference on it exists, 0 when freed.
8861  */
8862 static int
8863 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
8864 {
8865         struct mlx5_priv *priv = dev->data->dev_private;
8866         struct mlx5_dev_ctx_shared *sh = priv->sh;
8867         struct mlx5_flow_default_miss_resource *cache_resource =
8868                         &sh->default_miss;
8869
8870         MLX5_ASSERT(cache_resource->action);
8871         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
8872                         (void *)cache_resource->action,
8873                         rte_atomic32_read(&cache_resource->refcnt));
8874         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8875                 claim_zero(mlx5_glue->destroy_flow_action
8876                                 (cache_resource->action));
8877                 DRV_LOG(DEBUG, "default miss resource %p: removed",
8878                                 (void *)cache_resource->action);
8879                 return 0;
8880         }
8881         return 1;
8882 }
8883
8884 /**
8885  * Release a modify-header resource.
8886  *
8887  * @param handle
8888  *   Pointer to mlx5_flow_handle.
8889  *
8890  * @return
8891  *   1 while a reference on it exists, 0 when freed.
8892  */
8893 static int
8894 flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
8895 {
8896         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
8897                                                         handle->dvh.modify_hdr;
8898
8899         MLX5_ASSERT(cache_resource->action);
8900         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
8901                 (void *)cache_resource,
8902                 rte_atomic32_read(&cache_resource->refcnt));
8903         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8904                 claim_zero(mlx5_flow_os_destroy_flow_action
8905                                                 (cache_resource->action));
8906                 LIST_REMOVE(cache_resource, next);
8907                 rte_free(cache_resource);
8908                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
8909                         (void *)cache_resource);
8910                 return 0;
8911         }
8912         return 1;
8913 }
8914
8915 /**
8916  * Release port ID action resource.
8917  *
8918  * @param dev
8919  *   Pointer to Ethernet device.
8920  * @param handle
8921  *   Pointer to mlx5_flow_handle.
8922  *
8923  * @return
8924  *   1 while a reference on it exists, 0 when freed.
8925  */
8926 static int
8927 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
8928                                         struct mlx5_flow_handle *handle)
8929 {
8930         struct mlx5_priv *priv = dev->data->dev_private;
8931         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
8932         uint32_t idx = handle->rix_port_id_action;
8933
8934         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8935                                         idx);
8936         if (!cache_resource)
8937                 return 0;
8938         MLX5_ASSERT(cache_resource->action);
8939         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
8940                 (void *)cache_resource,
8941                 rte_atomic32_read(&cache_resource->refcnt));
8942         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8943                 claim_zero(mlx5_flow_os_destroy_flow_action
8944                                                 (cache_resource->action));
8945                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8946                              &priv->sh->port_id_action_list, idx,
8947                              cache_resource, next);
8948                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
8949                 DRV_LOG(DEBUG, "port id action resource %p: removed",
8950                         (void *)cache_resource);
8951                 return 0;
8952         }
8953         return 1;
8954 }
8955
8956 /**
8957  * Release push vlan action resource.
8958  *
8959  * @param dev
8960  *   Pointer to Ethernet device.
8961  * @param handle
8962  *   Pointer to mlx5_flow_handle.
8963  *
8964  * @return
8965  *   1 while a reference on it exists, 0 when freed.
8966  */
8967 static int
8968 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
8969                                           struct mlx5_flow_handle *handle)
8970 {
8971         struct mlx5_priv *priv = dev->data->dev_private;
8972         uint32_t idx = handle->dvh.rix_push_vlan;
8973         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
8974
8975         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
8976                                         idx);
8977         if (!cache_resource)
8978                 return 0;
8979         MLX5_ASSERT(cache_resource->action);
8980         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
8981                 (void *)cache_resource,
8982                 rte_atomic32_read(&cache_resource->refcnt));
8983         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8984                 claim_zero(mlx5_flow_os_destroy_flow_action
8985                                                 (cache_resource->action));
8986                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
8987                              &priv->sh->push_vlan_action_list, idx,
8988                              cache_resource, next);
8989                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
8990                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
8991                         (void *)cache_resource);
8992                 return 0;
8993         }
8994         return 1;
8995 }
8996
8997 /**
8998  * Release the fate resource.
8999  *
9000  * @param dev
9001  *   Pointer to Ethernet device.
9002  * @param handle
9003  *   Pointer to mlx5_flow_handle.
9004  */
9005 static void
9006 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
9007                                struct mlx5_flow_handle *handle)
9008 {
9009         if (!handle->rix_fate)
9010                 return;
9011         switch (handle->fate_action) {
9012         case MLX5_FLOW_FATE_DROP:
9013                 mlx5_hrxq_drop_release(dev);
9014                 break;
9015         case MLX5_FLOW_FATE_QUEUE:
9016                 mlx5_hrxq_release(dev, handle->rix_hrxq);
9017                 break;
9018         case MLX5_FLOW_FATE_JUMP:
9019                 flow_dv_jump_tbl_resource_release(dev, handle);
9020                 break;
9021         case MLX5_FLOW_FATE_PORT_ID:
9022                 flow_dv_port_id_action_resource_release(dev, handle);
9023                 break;
9024         case MLX5_FLOW_FATE_DEFAULT_MISS:
9025                 flow_dv_default_miss_resource_release(dev);
9026                 break;
9027         default:
9028                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
9029                 break;
9030         }
9031         handle->rix_fate = 0;
9032 }
9033
9034 /**
9035  * Remove the flow from the NIC but keeps it in memory.
9036  * Lock free, (mutex should be acquired by caller).
9037  *
9038  * @param[in] dev
9039  *   Pointer to Ethernet device.
9040  * @param[in, out] flow
9041  *   Pointer to flow structure.
9042  */
9043 static void
9044 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9045 {
9046         struct mlx5_flow_handle *dh;
9047         uint32_t handle_idx;
9048         struct mlx5_priv *priv = dev->data->dev_private;
9049
9050         if (!flow)
9051                 return;
9052         handle_idx = flow->dev_handles;
9053         while (handle_idx) {
9054                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9055                                     handle_idx);
9056                 if (!dh)
9057                         return;
9058                 if (dh->drv_flow) {
9059                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
9060                         dh->drv_flow = NULL;
9061                 }
9062                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
9063                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
9064                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
9065                         flow_dv_fate_resource_release(dev, dh);
9066                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
9067                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
9068                 handle_idx = dh->next.next;
9069         }
9070 }
9071
9072 /**
9073  * Remove the flow from the NIC and the memory.
9074  * Lock free, (mutex should be acquired by caller).
9075  *
9076  * @param[in] dev
9077  *   Pointer to the Ethernet device structure.
9078  * @param[in, out] flow
9079  *   Pointer to flow structure.
9080  */
9081 static void
9082 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9083 {
9084         struct mlx5_flow_handle *dev_handle;
9085         struct mlx5_priv *priv = dev->data->dev_private;
9086
9087         if (!flow)
9088                 return;
9089         __flow_dv_remove(dev, flow);
9090         if (flow->counter) {
9091                 flow_dv_counter_release(dev, flow->counter);
9092                 flow->counter = 0;
9093         }
9094         if (flow->meter) {
9095                 struct mlx5_flow_meter *fm;
9096
9097                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
9098                                     flow->meter);
9099                 if (fm)
9100                         mlx5_flow_meter_detach(fm);
9101                 flow->meter = 0;
9102         }
9103         while (flow->dev_handles) {
9104                 uint32_t tmp_idx = flow->dev_handles;
9105
9106                 dev_handle = mlx5_ipool_get(priv->sh->ipool
9107                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
9108                 if (!dev_handle)
9109                         return;
9110                 flow->dev_handles = dev_handle->next.next;
9111                 if (dev_handle->dvh.matcher)
9112                         flow_dv_matcher_release(dev, dev_handle);
9113                 if (dev_handle->dvh.rix_encap_decap)
9114                         flow_dv_encap_decap_resource_release(dev, dev_handle);
9115                 if (dev_handle->dvh.modify_hdr)
9116                         flow_dv_modify_hdr_resource_release(dev_handle);
9117                 if (dev_handle->dvh.rix_push_vlan)
9118                         flow_dv_push_vlan_action_resource_release(dev,
9119                                                                   dev_handle);
9120                 if (dev_handle->dvh.rix_tag)
9121                         flow_dv_tag_release(dev,
9122                                             dev_handle->dvh.rix_tag);
9123                 flow_dv_fate_resource_release(dev, dev_handle);
9124                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9125                            tmp_idx);
9126         }
9127 }
9128
9129 /**
9130  * Query a dv flow  rule for its statistics via devx.
9131  *
9132  * @param[in] dev
9133  *   Pointer to Ethernet device.
9134  * @param[in] flow
9135  *   Pointer to the sub flow.
9136  * @param[out] data
9137  *   data retrieved by the query.
9138  * @param[out] error
9139  *   Perform verbose error reporting if not NULL.
9140  *
9141  * @return
9142  *   0 on success, a negative errno value otherwise and rte_errno is set.
9143  */
9144 static int
9145 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
9146                     void *data, struct rte_flow_error *error)
9147 {
9148         struct mlx5_priv *priv = dev->data->dev_private;
9149         struct rte_flow_query_count *qc = data;
9150
9151         if (!priv->config.devx)
9152                 return rte_flow_error_set(error, ENOTSUP,
9153                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9154                                           NULL,
9155                                           "counters are not supported");
9156         if (flow->counter) {
9157                 uint64_t pkts, bytes;
9158                 struct mlx5_flow_counter *cnt;
9159
9160                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
9161                                                  NULL);
9162                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
9163                                                &bytes);
9164
9165                 if (err)
9166                         return rte_flow_error_set(error, -err,
9167                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9168                                         NULL, "cannot read counters");
9169                 qc->hits_set = 1;
9170                 qc->bytes_set = 1;
9171                 qc->hits = pkts - cnt->hits;
9172                 qc->bytes = bytes - cnt->bytes;
9173                 if (qc->reset) {
9174                         cnt->hits = pkts;
9175                         cnt->bytes = bytes;
9176                 }
9177                 return 0;
9178         }
9179         return rte_flow_error_set(error, EINVAL,
9180                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9181                                   NULL,
9182                                   "counters are not available");
9183 }
9184
9185 /**
9186  * Query a flow.
9187  *
9188  * @see rte_flow_query()
9189  * @see rte_flow_ops
9190  */
9191 static int
9192 flow_dv_query(struct rte_eth_dev *dev,
9193               struct rte_flow *flow __rte_unused,
9194               const struct rte_flow_action *actions __rte_unused,
9195               void *data __rte_unused,
9196               struct rte_flow_error *error __rte_unused)
9197 {
9198         int ret = -EINVAL;
9199
9200         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
9201                 switch (actions->type) {
9202                 case RTE_FLOW_ACTION_TYPE_VOID:
9203                         break;
9204                 case RTE_FLOW_ACTION_TYPE_COUNT:
9205                         ret = flow_dv_query_count(dev, flow, data, error);
9206                         break;
9207                 default:
9208                         return rte_flow_error_set(error, ENOTSUP,
9209                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9210                                                   actions,
9211                                                   "action not supported");
9212                 }
9213         }
9214         return ret;
9215 }
9216
9217 /**
9218  * Destroy the meter table set.
9219  * Lock free, (mutex should be acquired by caller).
9220  *
9221  * @param[in] dev
9222  *   Pointer to Ethernet device.
9223  * @param[in] tbl
9224  *   Pointer to the meter table set.
9225  *
9226  * @return
9227  *   Always 0.
9228  */
9229 static int
9230 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
9231                         struct mlx5_meter_domains_infos *tbl)
9232 {
9233         struct mlx5_priv *priv = dev->data->dev_private;
9234         struct mlx5_meter_domains_infos *mtd =
9235                                 (struct mlx5_meter_domains_infos *)tbl;
9236
9237         if (!mtd || !priv->config.dv_flow_en)
9238                 return 0;
9239         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
9240                 claim_zero(mlx5_flow_os_destroy_flow
9241                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
9242         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
9243                 claim_zero(mlx5_flow_os_destroy_flow
9244                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
9245         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
9246                 claim_zero(mlx5_flow_os_destroy_flow
9247                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
9248         if (mtd->egress.color_matcher)
9249                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9250                            (mtd->egress.color_matcher));
9251         if (mtd->egress.any_matcher)
9252                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9253                            (mtd->egress.any_matcher));
9254         if (mtd->egress.tbl)
9255                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
9256         if (mtd->egress.sfx_tbl)
9257                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
9258         if (mtd->ingress.color_matcher)
9259                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9260                            (mtd->ingress.color_matcher));
9261         if (mtd->ingress.any_matcher)
9262                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9263                            (mtd->ingress.any_matcher));
9264         if (mtd->ingress.tbl)
9265                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
9266         if (mtd->ingress.sfx_tbl)
9267                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
9268         if (mtd->transfer.color_matcher)
9269                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9270                            (mtd->transfer.color_matcher));
9271         if (mtd->transfer.any_matcher)
9272                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9273                            (mtd->transfer.any_matcher));
9274         if (mtd->transfer.tbl)
9275                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
9276         if (mtd->transfer.sfx_tbl)
9277                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
9278         if (mtd->drop_actn)
9279                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
9280         rte_free(mtd);
9281         return 0;
9282 }
9283
9284 /* Number of meter flow actions, count and jump or count and drop. */
9285 #define METER_ACTIONS 2
9286
9287 /**
9288  * Create specify domain meter table and suffix table.
9289  *
9290  * @param[in] dev
9291  *   Pointer to Ethernet device.
9292  * @param[in,out] mtb
9293  *   Pointer to DV meter table set.
9294  * @param[in] egress
9295  *   Table attribute.
9296  * @param[in] transfer
9297  *   Table attribute.
9298  * @param[in] color_reg_c_idx
9299  *   Reg C index for color match.
9300  *
9301  * @return
9302  *   0 on success, -1 otherwise and rte_errno is set.
9303  */
9304 static int
9305 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
9306                            struct mlx5_meter_domains_infos *mtb,
9307                            uint8_t egress, uint8_t transfer,
9308                            uint32_t color_reg_c_idx)
9309 {
9310         struct mlx5_priv *priv = dev->data->dev_private;
9311         struct mlx5_dev_ctx_shared *sh = priv->sh;
9312         struct mlx5_flow_dv_match_params mask = {
9313                 .size = sizeof(mask.buf),
9314         };
9315         struct mlx5_flow_dv_match_params value = {
9316                 .size = sizeof(value.buf),
9317         };
9318         struct mlx5dv_flow_matcher_attr dv_attr = {
9319                 .type = IBV_FLOW_ATTR_NORMAL,
9320                 .priority = 0,
9321                 .match_criteria_enable = 0,
9322                 .match_mask = (void *)&mask,
9323         };
9324         void *actions[METER_ACTIONS];
9325         struct mlx5_meter_domain_info *dtb;
9326         struct rte_flow_error error;
9327         int i = 0;
9328         int ret;
9329
9330         if (transfer)
9331                 dtb = &mtb->transfer;
9332         else if (egress)
9333                 dtb = &mtb->egress;
9334         else
9335                 dtb = &mtb->ingress;
9336         /* Create the meter table with METER level. */
9337         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
9338                                             egress, transfer, &error);
9339         if (!dtb->tbl) {
9340                 DRV_LOG(ERR, "Failed to create meter policer table.");
9341                 return -1;
9342         }
9343         /* Create the meter suffix table with SUFFIX level. */
9344         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
9345                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
9346                                             egress, transfer, &error);
9347         if (!dtb->sfx_tbl) {
9348                 DRV_LOG(ERR, "Failed to create meter suffix table.");
9349                 return -1;
9350         }
9351         /* Create matchers, Any and Color. */
9352         dv_attr.priority = 3;
9353         dv_attr.match_criteria_enable = 0;
9354         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9355                                                &dtb->any_matcher);
9356         if (ret) {
9357                 DRV_LOG(ERR, "Failed to create meter"
9358                              " policer default matcher.");
9359                 goto error_exit;
9360         }
9361         dv_attr.priority = 0;
9362         dv_attr.match_criteria_enable =
9363                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9364         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
9365                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
9366         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9367                                                &dtb->color_matcher);
9368         if (ret) {
9369                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
9370                 goto error_exit;
9371         }
9372         if (mtb->count_actns[RTE_MTR_DROPPED])
9373                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
9374         actions[i++] = mtb->drop_actn;
9375         /* Default rule: lowest priority, match any, actions: drop. */
9376         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
9377                                        actions,
9378                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
9379         if (ret) {
9380                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
9381                 goto error_exit;
9382         }
9383         return 0;
9384 error_exit:
9385         return -1;
9386 }
9387
9388 /**
9389  * Create the needed meter and suffix tables.
9390  * Lock free, (mutex should be acquired by caller).
9391  *
9392  * @param[in] dev
9393  *   Pointer to Ethernet device.
9394  * @param[in] fm
9395  *   Pointer to the flow meter.
9396  *
9397  * @return
9398  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
9399  */
9400 static struct mlx5_meter_domains_infos *
9401 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
9402                        const struct mlx5_flow_meter *fm)
9403 {
9404         struct mlx5_priv *priv = dev->data->dev_private;
9405         struct mlx5_meter_domains_infos *mtb;
9406         int ret;
9407         int i;
9408
9409         if (!priv->mtr_en) {
9410                 rte_errno = ENOTSUP;
9411                 return NULL;
9412         }
9413         mtb = rte_calloc(__func__, 1, sizeof(*mtb), 0);
9414         if (!mtb) {
9415                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
9416                 return NULL;
9417         }
9418         /* Create meter count actions */
9419         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
9420                 struct mlx5_flow_counter *cnt;
9421                 if (!fm->policer_stats.cnt[i])
9422                         continue;
9423                 cnt = flow_dv_counter_get_by_idx(dev,
9424                       fm->policer_stats.cnt[i], NULL);
9425                 mtb->count_actns[i] = cnt->action;
9426         }
9427         /* Create drop action. */
9428         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
9429         if (ret) {
9430                 DRV_LOG(ERR, "Failed to create drop action.");
9431                 goto error_exit;
9432         }
9433         /* Egress meter table. */
9434         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
9435         if (ret) {
9436                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
9437                 goto error_exit;
9438         }
9439         /* Ingress meter table. */
9440         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
9441         if (ret) {
9442                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
9443                 goto error_exit;
9444         }
9445         /* FDB meter table. */
9446         if (priv->config.dv_esw_en) {
9447                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
9448                                                  priv->mtr_color_reg);
9449                 if (ret) {
9450                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
9451                         goto error_exit;
9452                 }
9453         }
9454         return mtb;
9455 error_exit:
9456         flow_dv_destroy_mtr_tbl(dev, mtb);
9457         return NULL;
9458 }
9459
9460 /**
9461  * Destroy domain policer rule.
9462  *
9463  * @param[in] dt
9464  *   Pointer to domain table.
9465  */
9466 static void
9467 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
9468 {
9469         int i;
9470
9471         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9472                 if (dt->policer_rules[i]) {
9473                         claim_zero(mlx5_flow_os_destroy_flow
9474                                    (dt->policer_rules[i]));
9475                         dt->policer_rules[i] = NULL;
9476                 }
9477         }
9478         if (dt->jump_actn) {
9479                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
9480                 dt->jump_actn = NULL;
9481         }
9482 }
9483
9484 /**
9485  * Destroy policer rules.
9486  *
9487  * @param[in] dev
9488  *   Pointer to Ethernet device.
9489  * @param[in] fm
9490  *   Pointer to flow meter structure.
9491  * @param[in] attr
9492  *   Pointer to flow attributes.
9493  *
9494  * @return
9495  *   Always 0.
9496  */
9497 static int
9498 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
9499                               const struct mlx5_flow_meter *fm,
9500                               const struct rte_flow_attr *attr)
9501 {
9502         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
9503
9504         if (!mtb)
9505                 return 0;
9506         if (attr->egress)
9507                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
9508         if (attr->ingress)
9509                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
9510         if (attr->transfer)
9511                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
9512         return 0;
9513 }
9514
9515 /**
9516  * Create specify domain meter policer rule.
9517  *
9518  * @param[in] fm
9519  *   Pointer to flow meter structure.
9520  * @param[in] mtb
9521  *   Pointer to DV meter table set.
9522  * @param[in] mtr_reg_c
9523  *   Color match REG_C.
9524  *
9525  * @return
9526  *   0 on success, -1 otherwise.
9527  */
9528 static int
9529 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
9530                                     struct mlx5_meter_domain_info *dtb,
9531                                     uint8_t mtr_reg_c)
9532 {
9533         struct mlx5_flow_dv_match_params matcher = {
9534                 .size = sizeof(matcher.buf),
9535         };
9536         struct mlx5_flow_dv_match_params value = {
9537                 .size = sizeof(value.buf),
9538         };
9539         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9540         void *actions[METER_ACTIONS];
9541         int i;
9542         int ret = 0;
9543
9544         /* Create jump action. */
9545         if (!dtb->jump_actn)
9546                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
9547                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
9548         if (ret) {
9549                 DRV_LOG(ERR, "Failed to create policer jump action.");
9550                 goto error;
9551         }
9552         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9553                 int j = 0;
9554
9555                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
9556                                        rte_col_2_mlx5_col(i), UINT8_MAX);
9557                 if (mtb->count_actns[i])
9558                         actions[j++] = mtb->count_actns[i];
9559                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
9560                         actions[j++] = mtb->drop_actn;
9561                 else
9562                         actions[j++] = dtb->jump_actn;
9563                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
9564                                                (void *)&value, j, actions,
9565                                                &dtb->policer_rules[i]);
9566                 if (ret) {
9567                         DRV_LOG(ERR, "Failed to create policer rule.");
9568                         goto error;
9569                 }
9570         }
9571         return 0;
9572 error:
9573         rte_errno = errno;
9574         return -1;
9575 }
9576
9577 /**
9578  * Create policer rules.
9579  *
9580  * @param[in] dev
9581  *   Pointer to Ethernet device.
9582  * @param[in] fm
9583  *   Pointer to flow meter structure.
9584  * @param[in] attr
9585  *   Pointer to flow attributes.
9586  *
9587  * @return
9588  *   0 on success, -1 otherwise.
9589  */
9590 static int
9591 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
9592                              struct mlx5_flow_meter *fm,
9593                              const struct rte_flow_attr *attr)
9594 {
9595         struct mlx5_priv *priv = dev->data->dev_private;
9596         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9597         int ret;
9598
9599         if (attr->egress) {
9600                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
9601                                                 priv->mtr_color_reg);
9602                 if (ret) {
9603                         DRV_LOG(ERR, "Failed to create egress policer.");
9604                         goto error;
9605                 }
9606         }
9607         if (attr->ingress) {
9608                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
9609                                                 priv->mtr_color_reg);
9610                 if (ret) {
9611                         DRV_LOG(ERR, "Failed to create ingress policer.");
9612                         goto error;
9613                 }
9614         }
9615         if (attr->transfer) {
9616                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
9617                                                 priv->mtr_color_reg);
9618                 if (ret) {
9619                         DRV_LOG(ERR, "Failed to create transfer policer.");
9620                         goto error;
9621                 }
9622         }
9623         return 0;
9624 error:
9625         flow_dv_destroy_policer_rules(dev, fm, attr);
9626         return -1;
9627 }
9628
9629 /**
9630  * Query a devx counter.
9631  *
9632  * @param[in] dev
9633  *   Pointer to the Ethernet device structure.
9634  * @param[in] cnt
9635  *   Index to the flow counter.
9636  * @param[in] clear
9637  *   Set to clear the counter statistics.
9638  * @param[out] pkts
9639  *   The statistics value of packets.
9640  * @param[out] bytes
9641  *   The statistics value of bytes.
9642  *
9643  * @return
9644  *   0 on success, otherwise return -1.
9645  */
9646 static int
9647 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
9648                       uint64_t *pkts, uint64_t *bytes)
9649 {
9650         struct mlx5_priv *priv = dev->data->dev_private;
9651         struct mlx5_flow_counter *cnt;
9652         uint64_t inn_pkts, inn_bytes;
9653         int ret;
9654
9655         if (!priv->config.devx)
9656                 return -1;
9657
9658         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
9659         if (ret)
9660                 return -1;
9661         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
9662         *pkts = inn_pkts - cnt->hits;
9663         *bytes = inn_bytes - cnt->bytes;
9664         if (clear) {
9665                 cnt->hits = inn_pkts;
9666                 cnt->bytes = inn_bytes;
9667         }
9668         return 0;
9669 }
9670
9671 /**
9672  * Get aged-out flows.
9673  *
9674  * @param[in] dev
9675  *   Pointer to the Ethernet device structure.
9676  * @param[in] context
9677  *   The address of an array of pointers to the aged-out flows contexts.
9678  * @param[in] nb_contexts
9679  *   The length of context array pointers.
9680  * @param[out] error
9681  *   Perform verbose error reporting if not NULL. Initialized in case of
9682  *   error only.
9683  *
9684  * @return
9685  *   how many contexts get in success, otherwise negative errno value.
9686  *   if nb_contexts is 0, return the amount of all aged contexts.
9687  *   if nb_contexts is not 0 , return the amount of aged flows reported
9688  *   in the context array.
9689  * @note: only stub for now
9690  */
9691 static int
9692 flow_get_aged_flows(struct rte_eth_dev *dev,
9693                     void **context,
9694                     uint32_t nb_contexts,
9695                     struct rte_flow_error *error)
9696 {
9697         struct mlx5_priv *priv = dev->data->dev_private;
9698         struct mlx5_age_info *age_info;
9699         struct mlx5_age_param *age_param;
9700         struct mlx5_flow_counter *counter;
9701         int nb_flows = 0;
9702
9703         if (nb_contexts && !context)
9704                 return rte_flow_error_set(error, EINVAL,
9705                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9706                                           NULL,
9707                                           "Should assign at least one flow or"
9708                                           " context to get if nb_contexts != 0");
9709         age_info = GET_PORT_AGE_INFO(priv);
9710         rte_spinlock_lock(&age_info->aged_sl);
9711         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
9712                 nb_flows++;
9713                 if (nb_contexts) {
9714                         age_param = MLX5_CNT_TO_AGE(counter);
9715                         context[nb_flows - 1] = age_param->context;
9716                         if (!(--nb_contexts))
9717                                 break;
9718                 }
9719         }
9720         rte_spinlock_unlock(&age_info->aged_sl);
9721         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
9722         return nb_flows;
9723 }
9724
9725 /*
9726  * Mutex-protected thunk to lock-free  __flow_dv_translate().
9727  */
9728 static int
9729 flow_dv_translate(struct rte_eth_dev *dev,
9730                   struct mlx5_flow *dev_flow,
9731                   const struct rte_flow_attr *attr,
9732                   const struct rte_flow_item items[],
9733                   const struct rte_flow_action actions[],
9734                   struct rte_flow_error *error)
9735 {
9736         int ret;
9737
9738         flow_dv_shared_lock(dev);
9739         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
9740         flow_dv_shared_unlock(dev);
9741         return ret;
9742 }
9743
9744 /*
9745  * Mutex-protected thunk to lock-free  __flow_dv_apply().
9746  */
9747 static int
9748 flow_dv_apply(struct rte_eth_dev *dev,
9749               struct rte_flow *flow,
9750               struct rte_flow_error *error)
9751 {
9752         int ret;
9753
9754         flow_dv_shared_lock(dev);
9755         ret = __flow_dv_apply(dev, flow, error);
9756         flow_dv_shared_unlock(dev);
9757         return ret;
9758 }
9759
9760 /*
9761  * Mutex-protected thunk to lock-free __flow_dv_remove().
9762  */
9763 static void
9764 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9765 {
9766         flow_dv_shared_lock(dev);
9767         __flow_dv_remove(dev, flow);
9768         flow_dv_shared_unlock(dev);
9769 }
9770
9771 /*
9772  * Mutex-protected thunk to lock-free __flow_dv_destroy().
9773  */
9774 static void
9775 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9776 {
9777         flow_dv_shared_lock(dev);
9778         __flow_dv_destroy(dev, flow);
9779         flow_dv_shared_unlock(dev);
9780 }
9781
9782 /*
9783  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
9784  */
9785 static uint32_t
9786 flow_dv_counter_allocate(struct rte_eth_dev *dev)
9787 {
9788         uint32_t cnt;
9789
9790         flow_dv_shared_lock(dev);
9791         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
9792         flow_dv_shared_unlock(dev);
9793         return cnt;
9794 }
9795
9796 /*
9797  * Mutex-protected thunk to lock-free flow_dv_counter_release().
9798  */
9799 static void
9800 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
9801 {
9802         flow_dv_shared_lock(dev);
9803         flow_dv_counter_release(dev, cnt);
9804         flow_dv_shared_unlock(dev);
9805 }
9806
9807 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
9808         .validate = flow_dv_validate,
9809         .prepare = flow_dv_prepare,
9810         .translate = flow_dv_translate,
9811         .apply = flow_dv_apply,
9812         .remove = flow_dv_remove,
9813         .destroy = flow_dv_destroy,
9814         .query = flow_dv_query,
9815         .create_mtr_tbls = flow_dv_create_mtr_tbl,
9816         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
9817         .create_policer_rules = flow_dv_create_policer_rules,
9818         .destroy_policer_rules = flow_dv_destroy_policer_rules,
9819         .counter_alloc = flow_dv_counter_allocate,
9820         .counter_free = flow_dv_counter_free,
9821         .counter_query = flow_dv_counter_query,
9822         .get_aged_flows = flow_get_aged_flows,
9823 };
9824
9825 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */