drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24
  25 #include <mlx5_glue.h>
  26 #include <mlx5_devx_cmds.h>
  27 #include <mlx5_prm.h>
  28 #include <mlx5_malloc.h>
  29
  30 #include "mlx5_defs.h"
  31 #include "mlx5.h"
  32 #include "mlx5_common_os.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_flow_os.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 /* VLAN header definitions */
  54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  59
  60 union flow_dv_attr {
  61         struct {
  62                 uint32_t valid:1;
  63                 uint32_t ipv4:1;
  64                 uint32_t ipv6:1;
  65                 uint32_t tcp:1;
  66                 uint32_t udp:1;
  67                 uint32_t reserved:27;
  68         };
  69         uint32_t attr;
  70 };
  71
  72 static int
  73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  74                              struct mlx5_flow_tbl_resource *tbl);
  75
  76 static int
  77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  78
  79 static int
  80 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  81                                       uint32_t encap_decap_idx);
  82
  83 static int
  84 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  85                                         uint32_t port_id);
  86
  87 /**
  88  * Initialize flow attributes structure according to flow items' types.
  89  *
  90  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  91  * mode. For tunnel mode, the items to be modified are the outermost ones.
  92  *
  93  * @param[in] item
  94  *   Pointer to item specification.
  95  * @param[out] attr
  96  *   Pointer to flow attributes structure.
  97  * @param[in] dev_flow
  98  *   Pointer to the sub flow.
  99  * @param[in] tunnel_decap
 100  *   Whether action is after tunnel decapsulation.
 101  */
 102 static void
 103 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 104                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 105 {
 106         uint64_t layers = dev_flow->handle->layers;
 107
 108         /*
 109          * If layers is already initialized, it means this dev_flow is the
 110          * suffix flow, the layers flags is set by the prefix flow. Need to
 111          * use the layer flags from prefix flow as the suffix flow may not
 112          * have the user defined items as the flow is split.
 113          */
 114         if (layers) {
 115                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 116                         attr->ipv4 = 1;
 117                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 118                         attr->ipv6 = 1;
 119                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 120                         attr->tcp = 1;
 121                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 122                         attr->udp = 1;
 123                 attr->valid = 1;
 124                 return;
 125         }
 126         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 127                 uint8_t next_protocol = 0xff;
 128                 switch (item->type) {
 129                 case RTE_FLOW_ITEM_TYPE_GRE:
 130                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 132                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 133                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 case RTE_FLOW_ITEM_TYPE_MPLS:
 135                         if (tunnel_decap)
 136                                 attr->attr = 0;
 137                         break;
 138                 case RTE_FLOW_ITEM_TYPE_IPV4:
 139                         if (!attr->ipv6)
 140                                 attr->ipv4 = 1;
 141                         if (item->mask != NULL &&
 142                             ((const struct rte_flow_item_ipv4 *)
 143                             item->mask)->hdr.next_proto_id)
 144                                 next_protocol =
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->spec))->hdr.next_proto_id &
 147                                     ((const struct rte_flow_item_ipv4 *)
 148                                       (item->mask))->hdr.next_proto_id;
 149                         if ((next_protocol == IPPROTO_IPIP ||
 150                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 151                                 attr->attr = 0;
 152                         break;
 153                 case RTE_FLOW_ITEM_TYPE_IPV6:
 154                         if (!attr->ipv4)
 155                                 attr->ipv6 = 1;
 156                         if (item->mask != NULL &&
 157                             ((const struct rte_flow_item_ipv6 *)
 158                             item->mask)->hdr.proto)
 159                                 next_protocol =
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->spec))->hdr.proto &
 162                                     ((const struct rte_flow_item_ipv6 *)
 163                                       (item->mask))->hdr.proto;
 164                         if ((next_protocol == IPPROTO_IPIP ||
 165                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 166                                 attr->attr = 0;
 167                         break;
 168                 case RTE_FLOW_ITEM_TYPE_UDP:
 169                         if (!attr->tcp)
 170                                 attr->udp = 1;
 171                         break;
 172                 case RTE_FLOW_ITEM_TYPE_TCP:
 173                         if (!attr->udp)
 174                                 attr->tcp = 1;
 175                         break;
 176                 default:
 177                         break;
 178                 }
 179         }
 180         attr->valid = 1;
 181 }
 182
 183 /**
 184  * Convert rte_mtr_color to mlx5 color.
 185  *
 186  * @param[in] rcol
 187  *   rte_mtr_color.
 188  *
 189  * @return
 190  *   mlx5 color.
 191  */
 192 static int
 193 rte_col_2_mlx5_col(enum rte_color rcol)
 194 {
 195         switch (rcol) {
 196         case RTE_COLOR_GREEN:
 197                 return MLX5_FLOW_COLOR_GREEN;
 198         case RTE_COLOR_YELLOW:
 199                 return MLX5_FLOW_COLOR_YELLOW;
 200         case RTE_COLOR_RED:
 201                 return MLX5_FLOW_COLOR_RED;
 202         default:
 203                 break;
 204         }
 205         return MLX5_FLOW_COLOR_UNDEFINED;
 206 }
 207
 208 struct field_modify_info {
 209         uint32_t size; /* Size of field in protocol header, in bytes. */
 210         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 211         enum mlx5_modification_field id;
 212 };
 213
 214 struct field_modify_info modify_eth[] = {
 215         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 216         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 217         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 218         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 219         {0, 0, 0},
 220 };
 221
 222 struct field_modify_info modify_vlan_out_first_vid[] = {
 223         /* Size in bits !!! */
 224         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 225         {0, 0, 0},
 226 };
 227
 228 struct field_modify_info modify_ipv4[] = {
 229         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 230         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 231         {4, 12, MLX5_MODI_OUT_SIPV4},
 232         {4, 16, MLX5_MODI_OUT_DIPV4},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv6[] = {
 237         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 239         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 240         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 241         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 242         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 243         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 244         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 245         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 246         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 247         {0, 0, 0},
 248 };
 249
 250 struct field_modify_info modify_udp[] = {
 251         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 252         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 253         {0, 0, 0},
 254 };
 255
 256 struct field_modify_info modify_tcp[] = {
 257         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 258         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 259         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 260         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 261         {0, 0, 0},
 262 };
 263
 264 static void
 265 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 266                           uint8_t next_protocol, uint64_t *item_flags,
 267                           int *tunnel)
 268 {
 269         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 270                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 271         if (next_protocol == IPPROTO_IPIP) {
 272                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 273                 *tunnel = 1;
 274         }
 275         if (next_protocol == IPPROTO_IPV6) {
 276                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 277                 *tunnel = 1;
 278         }
 279 }
 280
 281 /**
 282  * Acquire the synchronizing object to protect multithreaded access
 283  * to shared dv context. Lock occurs only if context is actually
 284  * shared, i.e. we have multiport IB device and representors are
 285  * created.
 286  *
 287  * @param[in] dev
 288  *   Pointer to the rte_eth_dev structure.
 289  */
 290 static void
 291 flow_dv_shared_lock(struct rte_eth_dev *dev)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct mlx5_dev_ctx_shared *sh = priv->sh;
 295
 296         if (sh->dv_refcnt > 1) {
 297                 int ret;
 298
 299                 ret = pthread_mutex_lock(&sh->dv_mutex);
 300                 MLX5_ASSERT(!ret);
 301                 (void)ret;
 302         }
 303 }
 304
 305 static void
 306 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 307 {
 308         struct mlx5_priv *priv = dev->data->dev_private;
 309         struct mlx5_dev_ctx_shared *sh = priv->sh;
 310
 311         if (sh->dv_refcnt > 1) {
 312                 int ret;
 313
 314                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 315                 MLX5_ASSERT(!ret);
 316                 (void)ret;
 317         }
 318 }
 319
 320 /* Update VLAN's VID/PCP based on input rte_flow_action.
 321  *
 322  * @param[in] action
 323  *   Pointer to struct rte_flow_action.
 324  * @param[out] vlan
 325  *   Pointer to struct rte_vlan_hdr.
 326  */
 327 static void
 328 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 329                          struct rte_vlan_hdr *vlan)
 330 {
 331         uint16_t vlan_tci;
 332         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 333                 vlan_tci =
 334                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 335                                                action->conf)->vlan_pcp;
 336                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 337                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 338                 vlan->vlan_tci |= vlan_tci;
 339         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 340                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 341                 vlan->vlan_tci |= rte_be_to_cpu_16
 342                     (((const struct rte_flow_action_of_set_vlan_vid *)
 343                                              action->conf)->vlan_vid);
 344         }
 345 }
 346
 347 /**
 348  * Fetch 1, 2, 3 or 4 byte field from the byte array
 349  * and return as unsigned integer in host-endian format.
 350  *
 351  * @param[in] data
 352  *   Pointer to data array.
 353  * @param[in] size
 354  *   Size of field to extract.
 355  *
 356  * @return
 357  *   converted field in host endian format.
 358  */
 359 static inline uint32_t
 360 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 361 {
 362         uint32_t ret;
 363
 364         switch (size) {
 365         case 1:
 366                 ret = *data;
 367                 break;
 368         case 2:
 369                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 370                 break;
 371         case 3:
 372                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 373                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 374                 break;
 375         case 4:
 376                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 377                 break;
 378         default:
 379                 MLX5_ASSERT(false);
 380                 ret = 0;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 /**
 387  * Convert modify-header action to DV specification.
 388  *
 389  * Data length of each action is determined by provided field description
 390  * and the item mask. Data bit offset and width of each action is determined
 391  * by provided item mask.
 392  *
 393  * @param[in] item
 394  *   Pointer to item specification.
 395  * @param[in] field
 396  *   Pointer to field modification information.
 397  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 399  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 400  * @param[in] dcopy
 401  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 402  *   Negative offset value sets the same offset as source offset.
 403  *   size field is ignored, value is taken from source field.
 404  * @param[in,out] resource
 405  *   Pointer to the modify-header resource.
 406  * @param[in] type
 407  *   Type of modification.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_modify_action(struct rte_flow_item *item,
 416                               struct field_modify_info *field,
 417                               struct field_modify_info *dcopy,
 418                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 419                               uint32_t type, struct rte_flow_error *error)
 420 {
 421         uint32_t i = resource->actions_num;
 422         struct mlx5_modification_cmd *actions = resource->actions;
 423
 424         /*
 425          * The item and mask are provided in big-endian format.
 426          * The fields should be presented as in big-endian format either.
 427          * Mask must be always present, it defines the actual field width.
 428          */
 429         MLX5_ASSERT(item->mask);
 430         MLX5_ASSERT(field->size);
 431         do {
 432                 unsigned int size_b;
 433                 unsigned int off_b;
 434                 uint32_t mask;
 435                 uint32_t data;
 436
 437                 if (i >= MLX5_MAX_MODIFY_NUM)
 438                         return rte_flow_error_set(error, EINVAL,
 439                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 440                                  "too many items to modify");
 441                 /* Fetch variable byte size mask from the array. */
 442                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 443                                            field->offset, field->size);
 444                 if (!mask) {
 445                         ++field;
 446                         continue;
 447                 }
 448                 /* Deduce actual data width in bits from mask value. */
 449                 off_b = rte_bsf32(mask);
 450                 size_b = sizeof(uint32_t) * CHAR_BIT -
 451                          off_b - __builtin_clz(mask);
 452                 MLX5_ASSERT(size_b);
 453                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 454                 actions[i] = (struct mlx5_modification_cmd) {
 455                         .action_type = type,
 456                         .field = field->id,
 457                         .offset = off_b,
 458                         .length = size_b,
 459                 };
 460                 /* Convert entire record to expected big-endian format. */
 461                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 462                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 463                         MLX5_ASSERT(dcopy);
 464                         actions[i].dst_field = dcopy->id;
 465                         actions[i].dst_offset =
 466                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 467                         /* Convert entire record to big-endian format. */
 468                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 469                 } else {
 470                         MLX5_ASSERT(item->spec);
 471                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 472                                                    field->offset, field->size);
 473                         /* Shift out the trailing masked bits from data. */
 474                         data = (data & mask) >> off_b;
 475                         actions[i].data1 = rte_cpu_to_be_32(data);
 476                 }
 477                 ++i;
 478                 ++field;
 479         } while (field->size);
 480         if (resource->actions_num == i)
 481                 return rte_flow_error_set(error, EINVAL,
 482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 483                                           "invalid modification flow item");
 484         resource->actions_num = i;
 485         return 0;
 486 }
 487
 488 /**
 489  * Convert modify-header set IPv4 address action to DV specification.
 490  *
 491  * @param[in,out] resource
 492  *   Pointer to the modify-header resource.
 493  * @param[in] action
 494  *   Pointer to action specification.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_convert_action_modify_ipv4
 503                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 504                          const struct rte_flow_action *action,
 505                          struct rte_flow_error *error)
 506 {
 507         const struct rte_flow_action_set_ipv4 *conf =
 508                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 509         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 510         struct rte_flow_item_ipv4 ipv4;
 511         struct rte_flow_item_ipv4 ipv4_mask;
 512
 513         memset(&ipv4, 0, sizeof(ipv4));
 514         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 515         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 516                 ipv4.hdr.src_addr = conf->ipv4_addr;
 517                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 518         } else {
 519                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 520                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 521         }
 522         item.spec = &ipv4;
 523         item.mask = &ipv4_mask;
 524         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 525                                              MLX5_MODIFICATION_TYPE_SET, error);
 526 }
 527
 528 /**
 529  * Convert modify-header set IPv6 address action to DV specification.
 530  *
 531  * @param[in,out] resource
 532  *   Pointer to the modify-header resource.
 533  * @param[in] action
 534  *   Pointer to action specification.
 535  * @param[out] error
 536  *   Pointer to the error structure.
 537  *
 538  * @return
 539  *   0 on success, a negative errno value otherwise and rte_errno is set.
 540  */
 541 static int
 542 flow_dv_convert_action_modify_ipv6
 543                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 544                          const struct rte_flow_action *action,
 545                          struct rte_flow_error *error)
 546 {
 547         const struct rte_flow_action_set_ipv6 *conf =
 548                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 549         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 550         struct rte_flow_item_ipv6 ipv6;
 551         struct rte_flow_item_ipv6 ipv6_mask;
 552
 553         memset(&ipv6, 0, sizeof(ipv6));
 554         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 555         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 556                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.src_addr));
 558                 memcpy(&ipv6_mask.hdr.src_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 560                        sizeof(ipv6.hdr.src_addr));
 561         } else {
 562                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 563                        sizeof(ipv6.hdr.dst_addr));
 564                 memcpy(&ipv6_mask.hdr.dst_addr,
 565                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 566                        sizeof(ipv6.hdr.dst_addr));
 567         }
 568         item.spec = &ipv6;
 569         item.mask = &ipv6_mask;
 570         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 571                                              MLX5_MODIFICATION_TYPE_SET, error);
 572 }
 573
 574 /**
 575  * Convert modify-header set MAC address action to DV specification.
 576  *
 577  * @param[in,out] resource
 578  *   Pointer to the modify-header resource.
 579  * @param[in] action
 580  *   Pointer to action specification.
 581  * @param[out] error
 582  *   Pointer to the error structure.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_dv_convert_action_modify_mac
 589                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 590                          const struct rte_flow_action *action,
 591                          struct rte_flow_error *error)
 592 {
 593         const struct rte_flow_action_set_mac *conf =
 594                 (const struct rte_flow_action_set_mac *)(action->conf);
 595         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 596         struct rte_flow_item_eth eth;
 597         struct rte_flow_item_eth eth_mask;
 598
 599         memset(&eth, 0, sizeof(eth));
 600         memset(&eth_mask, 0, sizeof(eth_mask));
 601         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 602                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.src.addr_bytes));
 604                 memcpy(&eth_mask.src.addr_bytes,
 605                        &rte_flow_item_eth_mask.src.addr_bytes,
 606                        sizeof(eth_mask.src.addr_bytes));
 607         } else {
 608                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 609                        sizeof(eth.dst.addr_bytes));
 610                 memcpy(&eth_mask.dst.addr_bytes,
 611                        &rte_flow_item_eth_mask.dst.addr_bytes,
 612                        sizeof(eth_mask.dst.addr_bytes));
 613         }
 614         item.spec = &eth;
 615         item.mask = &eth_mask;
 616         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 617                                              MLX5_MODIFICATION_TYPE_SET, error);
 618 }
 619
 620 /**
 621  * Convert modify-header set VLAN VID action to DV specification.
 622  *
 623  * @param[in,out] resource
 624  *   Pointer to the modify-header resource.
 625  * @param[in] action
 626  *   Pointer to action specification.
 627  * @param[out] error
 628  *   Pointer to the error structure.
 629  *
 630  * @return
 631  *   0 on success, a negative errno value otherwise and rte_errno is set.
 632  */
 633 static int
 634 flow_dv_convert_action_modify_vlan_vid
 635                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 636                          const struct rte_flow_action *action,
 637                          struct rte_flow_error *error)
 638 {
 639         const struct rte_flow_action_of_set_vlan_vid *conf =
 640                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 641         int i = resource->actions_num;
 642         struct mlx5_modification_cmd *actions = resource->actions;
 643         struct field_modify_info *field = modify_vlan_out_first_vid;
 644
 645         if (i >= MLX5_MAX_MODIFY_NUM)
 646                 return rte_flow_error_set(error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 648                          "too many items to modify");
 649         actions[i] = (struct mlx5_modification_cmd) {
 650                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 651                 .field = field->id,
 652                 .length = field->size,
 653                 .offset = field->offset,
 654         };
 655         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 656         actions[i].data1 = conf->vlan_vid;
 657         actions[i].data1 = actions[i].data1 << 16;
 658         resource->actions_num = ++i;
 659         return 0;
 660 }
 661
 662 /**
 663  * Convert modify-header set TP action to DV specification.
 664  *
 665  * @param[in,out] resource
 666  *   Pointer to the modify-header resource.
 667  * @param[in] action
 668  *   Pointer to action specification.
 669  * @param[in] items
 670  *   Pointer to rte_flow_item objects list.
 671  * @param[in] attr
 672  *   Pointer to flow attributes structure.
 673  * @param[in] dev_flow
 674  *   Pointer to the sub flow.
 675  * @param[in] tunnel_decap
 676  *   Whether action is after tunnel decapsulation.
 677  * @param[out] error
 678  *   Pointer to the error structure.
 679  *
 680  * @return
 681  *   0 on success, a negative errno value otherwise and rte_errno is set.
 682  */
 683 static int
 684 flow_dv_convert_action_modify_tp
 685                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 686                          const struct rte_flow_action *action,
 687                          const struct rte_flow_item *items,
 688                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 689                          bool tunnel_decap, struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_set_tp *conf =
 692                 (const struct rte_flow_action_set_tp *)(action->conf);
 693         struct rte_flow_item item;
 694         struct rte_flow_item_udp udp;
 695         struct rte_flow_item_udp udp_mask;
 696         struct rte_flow_item_tcp tcp;
 697         struct rte_flow_item_tcp tcp_mask;
 698         struct field_modify_info *field;
 699
 700         if (!attr->valid)
 701                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 702         if (attr->udp) {
 703                 memset(&udp, 0, sizeof(udp));
 704                 memset(&udp_mask, 0, sizeof(udp_mask));
 705                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 706                         udp.hdr.src_port = conf->port;
 707                         udp_mask.hdr.src_port =
 708                                         rte_flow_item_udp_mask.hdr.src_port;
 709                 } else {
 710                         udp.hdr.dst_port = conf->port;
 711                         udp_mask.hdr.dst_port =
 712                                         rte_flow_item_udp_mask.hdr.dst_port;
 713                 }
 714                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 715                 item.spec = &udp;
 716                 item.mask = &udp_mask;
 717                 field = modify_udp;
 718         } else {
 719                 MLX5_ASSERT(attr->tcp);
 720                 memset(&tcp, 0, sizeof(tcp));
 721                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 722                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 723                         tcp.hdr.src_port = conf->port;
 724                         tcp_mask.hdr.src_port =
 725                                         rte_flow_item_tcp_mask.hdr.src_port;
 726                 } else {
 727                         tcp.hdr.dst_port = conf->port;
 728                         tcp_mask.hdr.dst_port =
 729                                         rte_flow_item_tcp_mask.hdr.dst_port;
 730                 }
 731                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 732                 item.spec = &tcp;
 733                 item.mask = &tcp_mask;
 734                 field = modify_tcp;
 735         }
 736         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 737                                              MLX5_MODIFICATION_TYPE_SET, error);
 738 }
 739
 740 /**
 741  * Convert modify-header set TTL action to DV specification.
 742  *
 743  * @param[in,out] resource
 744  *   Pointer to the modify-header resource.
 745  * @param[in] action
 746  *   Pointer to action specification.
 747  * @param[in] items
 748  *   Pointer to rte_flow_item objects list.
 749  * @param[in] attr
 750  *   Pointer to flow attributes structure.
 751  * @param[in] dev_flow
 752  *   Pointer to the sub flow.
 753  * @param[in] tunnel_decap
 754  *   Whether action is after tunnel decapsulation.
 755  * @param[out] error
 756  *   Pointer to the error structure.
 757  *
 758  * @return
 759  *   0 on success, a negative errno value otherwise and rte_errno is set.
 760  */
 761 static int
 762 flow_dv_convert_action_modify_ttl
 763                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 764                          const struct rte_flow_action *action,
 765                          const struct rte_flow_item *items,
 766                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 767                          bool tunnel_decap, struct rte_flow_error *error)
 768 {
 769         const struct rte_flow_action_set_ttl *conf =
 770                 (const struct rte_flow_action_set_ttl *)(action->conf);
 771         struct rte_flow_item item;
 772         struct rte_flow_item_ipv4 ipv4;
 773         struct rte_flow_item_ipv4 ipv4_mask;
 774         struct rte_flow_item_ipv6 ipv6;
 775         struct rte_flow_item_ipv6 ipv6_mask;
 776         struct field_modify_info *field;
 777
 778         if (!attr->valid)
 779                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 780         if (attr->ipv4) {
 781                 memset(&ipv4, 0, sizeof(ipv4));
 782                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 783                 ipv4.hdr.time_to_live = conf->ttl_value;
 784                 ipv4_mask.hdr.time_to_live = 0xFF;
 785                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 786                 item.spec = &ipv4;
 787                 item.mask = &ipv4_mask;
 788                 field = modify_ipv4;
 789         } else {
 790                 MLX5_ASSERT(attr->ipv6);
 791                 memset(&ipv6, 0, sizeof(ipv6));
 792                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 793                 ipv6.hdr.hop_limits = conf->ttl_value;
 794                 ipv6_mask.hdr.hop_limits = 0xFF;
 795                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 796                 item.spec = &ipv6;
 797                 item.mask = &ipv6_mask;
 798                 field = modify_ipv6;
 799         }
 800         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 801                                              MLX5_MODIFICATION_TYPE_SET, error);
 802 }
 803
 804 /**
 805  * Convert modify-header decrement TTL action to DV specification.
 806  *
 807  * @param[in,out] resource
 808  *   Pointer to the modify-header resource.
 809  * @param[in] action
 810  *   Pointer to action specification.
 811  * @param[in] items
 812  *   Pointer to rte_flow_item objects list.
 813  * @param[in] attr
 814  *   Pointer to flow attributes structure.
 815  * @param[in] dev_flow
 816  *   Pointer to the sub flow.
 817  * @param[in] tunnel_decap
 818  *   Whether action is after tunnel decapsulation.
 819  * @param[out] error
 820  *   Pointer to the error structure.
 821  *
 822  * @return
 823  *   0 on success, a negative errno value otherwise and rte_errno is set.
 824  */
 825 static int
 826 flow_dv_convert_action_modify_dec_ttl
 827                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 828                          const struct rte_flow_item *items,
 829                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 830                          bool tunnel_decap, struct rte_flow_error *error)
 831 {
 832         struct rte_flow_item item;
 833         struct rte_flow_item_ipv4 ipv4;
 834         struct rte_flow_item_ipv4 ipv4_mask;
 835         struct rte_flow_item_ipv6 ipv6;
 836         struct rte_flow_item_ipv6 ipv6_mask;
 837         struct field_modify_info *field;
 838
 839         if (!attr->valid)
 840                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 841         if (attr->ipv4) {
 842                 memset(&ipv4, 0, sizeof(ipv4));
 843                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 844                 ipv4.hdr.time_to_live = 0xFF;
 845                 ipv4_mask.hdr.time_to_live = 0xFF;
 846                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 847                 item.spec = &ipv4;
 848                 item.mask = &ipv4_mask;
 849                 field = modify_ipv4;
 850         } else {
 851                 MLX5_ASSERT(attr->ipv6);
 852                 memset(&ipv6, 0, sizeof(ipv6));
 853                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 854                 ipv6.hdr.hop_limits = 0xFF;
 855                 ipv6_mask.hdr.hop_limits = 0xFF;
 856                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 857                 item.spec = &ipv6;
 858                 item.mask = &ipv6_mask;
 859                 field = modify_ipv6;
 860         }
 861         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 862                                              MLX5_MODIFICATION_TYPE_ADD, error);
 863 }
 864
 865 /**
 866  * Convert modify-header increment/decrement TCP Sequence number
 867  * to DV specification.
 868  *
 869  * @param[in,out] resource
 870  *   Pointer to the modify-header resource.
 871  * @param[in] action
 872  *   Pointer to action specification.
 873  * @param[out] error
 874  *   Pointer to the error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_errno is set.
 878  */
 879 static int
 880 flow_dv_convert_action_modify_tcp_seq
 881                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 882                          const struct rte_flow_action *action,
 883                          struct rte_flow_error *error)
 884 {
 885         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 886         uint64_t value = rte_be_to_cpu_32(*conf);
 887         struct rte_flow_item item;
 888         struct rte_flow_item_tcp tcp;
 889         struct rte_flow_item_tcp tcp_mask;
 890
 891         memset(&tcp, 0, sizeof(tcp));
 892         memset(&tcp_mask, 0, sizeof(tcp_mask));
 893         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 894                 /*
 895                  * The HW has no decrement operation, only increment operation.
 896                  * To simulate decrement X from Y using increment operation
 897                  * we need to add UINT32_MAX X times to Y.
 898                  * Each adding of UINT32_MAX decrements Y by 1.
 899                  */
 900                 value *= UINT32_MAX;
 901         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 902         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 903         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 904         item.spec = &tcp;
 905         item.mask = &tcp_mask;
 906         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 907                                              MLX5_MODIFICATION_TYPE_ADD, error);
 908 }
 909
 910 /**
 911  * Convert modify-header increment/decrement TCP Acknowledgment number
 912  * to DV specification.
 913  *
 914  * @param[in,out] resource
 915  *   Pointer to the modify-header resource.
 916  * @param[in] action
 917  *   Pointer to action specification.
 918  * @param[out] error
 919  *   Pointer to the error structure.
 920  *
 921  * @return
 922  *   0 on success, a negative errno value otherwise and rte_errno is set.
 923  */
 924 static int
 925 flow_dv_convert_action_modify_tcp_ack
 926                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 927                          const struct rte_flow_action *action,
 928                          struct rte_flow_error *error)
 929 {
 930         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 931         uint64_t value = rte_be_to_cpu_32(*conf);
 932         struct rte_flow_item item;
 933         struct rte_flow_item_tcp tcp;
 934         struct rte_flow_item_tcp tcp_mask;
 935
 936         memset(&tcp, 0, sizeof(tcp));
 937         memset(&tcp_mask, 0, sizeof(tcp_mask));
 938         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 939                 /*
 940                  * The HW has no decrement operation, only increment operation.
 941                  * To simulate decrement X from Y using increment operation
 942                  * we need to add UINT32_MAX X times to Y.
 943                  * Each adding of UINT32_MAX decrements Y by 1.
 944                  */
 945                 value *= UINT32_MAX;
 946         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 947         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 948         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 949         item.spec = &tcp;
 950         item.mask = &tcp_mask;
 951         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 952                                              MLX5_MODIFICATION_TYPE_ADD, error);
 953 }
 954
 955 static enum mlx5_modification_field reg_to_field[] = {
 956         [REG_NON] = MLX5_MODI_OUT_NONE,
 957         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 958         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 959         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 960         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 961         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 962         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 963         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 964         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 965         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 966         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 967 };
 968
 969 /**
 970  * Convert register set to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_set_reg
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 989         struct mlx5_modification_cmd *actions = resource->actions;
 990         uint32_t i = resource->actions_num;
 991
 992         if (i >= MLX5_MAX_MODIFY_NUM)
 993                 return rte_flow_error_set(error, EINVAL,
 994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 995                                           "too many items to modify");
 996         MLX5_ASSERT(conf->id != REG_NON);
 997         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 998         actions[i] = (struct mlx5_modification_cmd) {
 999                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1000                 .field = reg_to_field[conf->id],
1001         };
1002         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1003         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1004         ++i;
1005         resource->actions_num = i;
1006         return 0;
1007 }
1008
1009 /**
1010  * Convert SET_TAG action to DV specification.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the rte_eth_dev structure.
1014  * @param[in,out] resource
1015  *   Pointer to the modify-header resource.
1016  * @param[in] conf
1017  *   Pointer to action specification.
1018  * @param[out] error
1019  *   Pointer to the error structure.
1020  *
1021  * @return
1022  *   0 on success, a negative errno value otherwise and rte_errno is set.
1023  */
1024 static int
1025 flow_dv_convert_action_set_tag
1026                         (struct rte_eth_dev *dev,
1027                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1028                          const struct rte_flow_action_set_tag *conf,
1029                          struct rte_flow_error *error)
1030 {
1031         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1032         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1033         struct rte_flow_item item = {
1034                 .spec = &data,
1035                 .mask = &mask,
1036         };
1037         struct field_modify_info reg_c_x[] = {
1038                 [1] = {0, 0, 0},
1039         };
1040         enum mlx5_modification_field reg_type;
1041         int ret;
1042
1043         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044         if (ret < 0)
1045                 return ret;
1046         MLX5_ASSERT(ret != REG_NON);
1047         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1048         reg_type = reg_to_field[ret];
1049         MLX5_ASSERT(reg_type > 0);
1050         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1051         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1052                                              MLX5_MODIFICATION_TYPE_SET, error);
1053 }
1054
1055 /**
1056  * Convert internal COPY_REG action to DV specification.
1057  *
1058  * @param[in] dev
1059  *   Pointer to the rte_eth_dev structure.
1060  * @param[in,out] res
1061  *   Pointer to the modify-header resource.
1062  * @param[in] action
1063  *   Pointer to action specification.
1064  * @param[out] error
1065  *   Pointer to the error structure.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 static int
1071 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1072                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1073                                  const struct rte_flow_action *action,
1074                                  struct rte_flow_error *error)
1075 {
1076         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1077         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1078         struct rte_flow_item item = {
1079                 .spec = NULL,
1080                 .mask = &mask,
1081         };
1082         struct field_modify_info reg_src[] = {
1083                 {4, 0, reg_to_field[conf->src]},
1084                 {0, 0, 0},
1085         };
1086         struct field_modify_info reg_dst = {
1087                 .offset = 0,
1088                 .id = reg_to_field[conf->dst],
1089         };
1090         /* Adjust reg_c[0] usage according to reported mask. */
1091         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1092                 struct mlx5_priv *priv = dev->data->dev_private;
1093                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1094
1095                 MLX5_ASSERT(reg_c0);
1096                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1097                 if (conf->dst == REG_C_0) {
1098                         /* Copy to reg_c[0], within mask only. */
1099                         reg_dst.offset = rte_bsf32(reg_c0);
1100                         /*
1101                          * Mask is ignoring the enianness, because
1102                          * there is no conversion in datapath.
1103                          */
1104 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1105                         /* Copy from destination lower bits to reg_c[0]. */
1106                         mask = reg_c0 >> reg_dst.offset;
1107 #else
1108                         /* Copy from destination upper bits to reg_c[0]. */
1109                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1110                                           rte_fls_u32(reg_c0));
1111 #endif
1112                 } else {
1113                         mask = rte_cpu_to_be_32(reg_c0);
1114 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1115                         /* Copy from reg_c[0] to destination lower bits. */
1116                         reg_dst.offset = 0;
1117 #else
1118                         /* Copy from reg_c[0] to destination upper bits. */
1119                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1120                                          (rte_fls_u32(reg_c0) -
1121                                           rte_bsf32(reg_c0));
1122 #endif
1123                 }
1124         }
1125         return flow_dv_convert_modify_action(&item,
1126                                              reg_src, &reg_dst, res,
1127                                              MLX5_MODIFICATION_TYPE_COPY,
1128                                              error);
1129 }
1130
1131 /**
1132  * Convert MARK action to DV specification. This routine is used
1133  * in extensive metadata only and requires metadata register to be
1134  * handled. In legacy mode hardware tag resource is engaged.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in] conf
1139  *   Pointer to MARK action specification.
1140  * @param[in,out] resource
1141  *   Pointer to the modify-header resource.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1150                             const struct rte_flow_action_mark *conf,
1151                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1152                             struct rte_flow_error *error)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1156                                            priv->sh->dv_mark_mask);
1157         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1158         struct rte_flow_item item = {
1159                 .spec = &data,
1160                 .mask = &mask,
1161         };
1162         struct field_modify_info reg_c_x[] = {
1163                 [1] = {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         } else if (attr->transfer) {
1488                 return rte_flow_error_set(error, ENOTSUP,
1489                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1490                                         "extended metadata feature "
1491                                         "should be enabled when "
1492                                         "meta item is requested "
1493                                         "with e-switch mode ");
1494         }
1495         if (!mask)
1496                 mask = &rte_flow_item_meta_mask;
1497         if (!mask->data)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1500                                         "mask cannot be zero");
1501
1502         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1503                                         (const uint8_t *)&nic_mask,
1504                                         sizeof(struct rte_flow_item_meta),
1505                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1506         return ret;
1507 }
1508
1509 /**
1510  * Validate TAG item.
1511  *
1512  * @param[in] dev
1513  *   Pointer to the rte_eth_dev structure.
1514  * @param[in] item
1515  *   Item specification.
1516  * @param[in] attr
1517  *   Attributes of flow that includes this item.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 static int
1525 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1526                           const struct rte_flow_item *item,
1527                           const struct rte_flow_attr *attr __rte_unused,
1528                           struct rte_flow_error *error)
1529 {
1530         const struct rte_flow_item_tag *spec = item->spec;
1531         const struct rte_flow_item_tag *mask = item->mask;
1532         const struct rte_flow_item_tag nic_mask = {
1533                 .data = RTE_BE32(UINT32_MAX),
1534                 .index = 0xff,
1535         };
1536         int ret;
1537
1538         if (!mlx5_flow_ext_mreg_supported(dev))
1539                 return rte_flow_error_set(error, ENOTSUP,
1540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1541                                           "extensive metadata register"
1542                                           " isn't supported");
1543         if (!spec)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546                                           item->spec,
1547                                           "data cannot be empty");
1548         if (!mask)
1549                 mask = &rte_flow_item_tag_mask;
1550         if (!mask->data)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                         "mask cannot be zero");
1554
1555         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1556                                         (const uint8_t *)&nic_mask,
1557                                         sizeof(struct rte_flow_item_tag),
1558                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1559         if (ret < 0)
1560                 return ret;
1561         if (mask->index != 0xff)
1562                 return rte_flow_error_set(error, EINVAL,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1564                                           "partial mask for tag index"
1565                                           " is not supported");
1566         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1567         if (ret < 0)
1568                 return ret;
1569         MLX5_ASSERT(ret != REG_NON);
1570         return 0;
1571 }
1572
1573 /**
1574  * Validate vport item.
1575  *
1576  * @param[in] dev
1577  *   Pointer to the rte_eth_dev structure.
1578  * @param[in] item
1579  *   Item specification.
1580  * @param[in] attr
1581  *   Attributes of flow that includes this item.
1582  * @param[in] item_flags
1583  *   Bit-fields that holds the items detected until now.
1584  * @param[out] error
1585  *   Pointer to error structure.
1586  *
1587  * @return
1588  *   0 on success, a negative errno value otherwise and rte_errno is set.
1589  */
1590 static int
1591 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1592                               const struct rte_flow_item *item,
1593                               const struct rte_flow_attr *attr,
1594                               uint64_t item_flags,
1595                               struct rte_flow_error *error)
1596 {
1597         const struct rte_flow_item_port_id *spec = item->spec;
1598         const struct rte_flow_item_port_id *mask = item->mask;
1599         const struct rte_flow_item_port_id switch_mask = {
1600                         .id = 0xffffffff,
1601         };
1602         struct mlx5_priv *esw_priv;
1603         struct mlx5_priv *dev_priv;
1604         int ret;
1605
1606         if (!attr->transfer)
1607                 return rte_flow_error_set(error, EINVAL,
1608                                           RTE_FLOW_ERROR_TYPE_ITEM,
1609                                           NULL,
1610                                           "match on port id is valid only"
1611                                           " when transfer flag is enabled");
1612         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1615                                           "multiple source ports are not"
1616                                           " supported");
1617         if (!mask)
1618                 mask = &switch_mask;
1619         if (mask->id != 0xffffffff)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622                                            mask,
1623                                            "no support for partial mask on"
1624                                            " \"id\" field");
1625         ret = mlx5_flow_item_acceptable
1626                                 (item, (const uint8_t *)mask,
1627                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1628                                  sizeof(struct rte_flow_item_port_id),
1629                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1630         if (ret)
1631                 return ret;
1632         if (!spec)
1633                 return 0;
1634         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635         if (!esw_priv)
1636                 return rte_flow_error_set(error, rte_errno,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "failed to obtain E-Switch info for"
1639                                           " port");
1640         dev_priv = mlx5_dev_to_eswitch_info(dev);
1641         if (!dev_priv)
1642                 return rte_flow_error_set(error, rte_errno,
1643                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644                                           NULL,
1645                                           "failed to obtain E-Switch info");
1646         if (esw_priv->domain_id != dev_priv->domain_id)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1649                                           "cannot match on a port from a"
1650                                           " different E-Switch");
1651         return 0;
1652 }
1653
1654 /**
1655  * Validate VLAN item.
1656  *
1657  * @param[in] item
1658  *   Item specification.
1659  * @param[in] item_flags
1660  *   Bit-fields that holds the items detected until now.
1661  * @param[in] dev
1662  *   Ethernet device flow is being created on.
1663  * @param[out] error
1664  *   Pointer to error structure.
1665  *
1666  * @return
1667  *   0 on success, a negative errno value otherwise and rte_errno is set.
1668  */
1669 static int
1670 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1671                            uint64_t item_flags,
1672                            struct rte_eth_dev *dev,
1673                            struct rte_flow_error *error)
1674 {
1675         const struct rte_flow_item_vlan *mask = item->mask;
1676         const struct rte_flow_item_vlan nic_mask = {
1677                 .tci = RTE_BE16(UINT16_MAX),
1678                 .inner_type = RTE_BE16(UINT16_MAX),
1679         };
1680         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1681         int ret;
1682         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1683                                         MLX5_FLOW_LAYER_INNER_L4) :
1684                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1685                                         MLX5_FLOW_LAYER_OUTER_L4);
1686         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1687                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1688
1689         if (item_flags & vlanm)
1690                 return rte_flow_error_set(error, EINVAL,
1691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1692                                           "multiple VLAN layers not supported");
1693         else if ((item_flags & l34m) != 0)
1694                 return rte_flow_error_set(error, EINVAL,
1695                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1696                                           "VLAN cannot follow L3/L4 layer");
1697         if (!mask)
1698                 mask = &rte_flow_item_vlan_mask;
1699         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1700                                         (const uint8_t *)&nic_mask,
1701                                         sizeof(struct rte_flow_item_vlan),
1702                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1703         if (ret)
1704                 return ret;
1705         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1706                 struct mlx5_priv *priv = dev->data->dev_private;
1707
1708                 if (priv->vmwa_context) {
1709                         /*
1710                          * Non-NULL context means we have a virtual machine
1711                          * and SR-IOV enabled, we have to create VLAN interface
1712                          * to make hypervisor to setup E-Switch vport
1713                          * context correctly. We avoid creating the multiple
1714                          * VLAN interfaces, so we cannot support VLAN tag mask.
1715                          */
1716                         return rte_flow_error_set(error, EINVAL,
1717                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1718                                                   item,
1719                                                   "VLAN tag mask is not"
1720                                                   " supported in virtual"
1721                                                   " environment");
1722                 }
1723         }
1724         return 0;
1725 }
1726
1727 /*
1728  * GTP flags are contained in 1 byte of the format:
1729  * -------------------------------------------
1730  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1731  * |-----------------------------------------|
1732  * | value | Version | PT | Res | E | S | PN |
1733  * -------------------------------------------
1734  *
1735  * Matching is supported only for GTP flags E, S, PN.
1736  */
1737 #define MLX5_GTP_FLAGS_MASK     0x07
1738
1739 /**
1740  * Validate GTP item.
1741  *
1742  * @param[in] dev
1743  *   Pointer to the rte_eth_dev structure.
1744  * @param[in] item
1745  *   Item specification.
1746  * @param[in] item_flags
1747  *   Bit-fields that holds the items detected until now.
1748  * @param[out] error
1749  *   Pointer to error structure.
1750  *
1751  * @return
1752  *   0 on success, a negative errno value otherwise and rte_errno is set.
1753  */
1754 static int
1755 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1756                           const struct rte_flow_item *item,
1757                           uint64_t item_flags,
1758                           struct rte_flow_error *error)
1759 {
1760         struct mlx5_priv *priv = dev->data->dev_private;
1761         const struct rte_flow_item_gtp *spec = item->spec;
1762         const struct rte_flow_item_gtp *mask = item->mask;
1763         const struct rte_flow_item_gtp nic_mask = {
1764                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1765                 .msg_type = 0xff,
1766                 .teid = RTE_BE32(0xffffffff),
1767         };
1768
1769         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1772                                           "GTP support is not enabled");
1773         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1776                                           "multiple tunnel layers not"
1777                                           " supported");
1778         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1779                 return rte_flow_error_set(error, EINVAL,
1780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1781                                           "no outer UDP layer found");
1782         if (!mask)
1783                 mask = &rte_flow_item_gtp_mask;
1784         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1785                 return rte_flow_error_set(error, ENOTSUP,
1786                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1787                                           "Match is supported for GTP"
1788                                           " flags only");
1789         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1790                                          (const uint8_t *)&nic_mask,
1791                                          sizeof(struct rte_flow_item_gtp),
1792                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1793 }
1794
1795 /**
1796  * Validate IPV4 item.
1797  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
1798  * add specific validation of fragment_offset field,
1799  *
1800  * @param[in] item
1801  *   Item specification.
1802  * @param[in] item_flags
1803  *   Bit-fields that holds the items detected until now.
1804  * @param[out] error
1805  *   Pointer to error structure.
1806  *
1807  * @return
1808  *   0 on success, a negative errno value otherwise and rte_errno is set.
1809  */
1810 static int
1811 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
1812                            uint64_t item_flags,
1813                            uint64_t last_item,
1814                            uint16_t ether_type,
1815                            struct rte_flow_error *error)
1816 {
1817         int ret;
1818         const struct rte_flow_item_ipv4 *spec = item->spec;
1819         const struct rte_flow_item_ipv4 *last = item->last;
1820         const struct rte_flow_item_ipv4 *mask = item->mask;
1821         rte_be16_t fragment_offset_spec = 0;
1822         rte_be16_t fragment_offset_last = 0;
1823         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
1824                 .hdr = {
1825                         .src_addr = RTE_BE32(0xffffffff),
1826                         .dst_addr = RTE_BE32(0xffffffff),
1827                         .type_of_service = 0xff,
1828                         .fragment_offset = RTE_BE16(0xffff),
1829                         .next_proto_id = 0xff,
1830                         .time_to_live = 0xff,
1831                 },
1832         };
1833
1834         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
1835                                            ether_type, &nic_ipv4_mask,
1836                                            MLX5_ITEM_RANGE_ACCEPTED, error);
1837         if (ret < 0)
1838                 return ret;
1839         if (spec && mask)
1840                 fragment_offset_spec = spec->hdr.fragment_offset &
1841                                        mask->hdr.fragment_offset;
1842         if (!fragment_offset_spec)
1843                 return 0;
1844         /*
1845          * spec and mask are valid, enforce using full mask to make sure the
1846          * complete value is used correctly.
1847          */
1848         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1849                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1850                 return rte_flow_error_set(error, EINVAL,
1851                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1852                                           item, "must use full mask for"
1853                                           " fragment_offset");
1854         /*
1855          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
1856          * indicating this is 1st fragment of fragmented packet.
1857          * This is not yet supported in MLX5, return appropriate error message.
1858          */
1859         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
1860                 return rte_flow_error_set(error, ENOTSUP,
1861                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1862                                           "match on first fragment not "
1863                                           "supported");
1864         if (fragment_offset_spec && !last)
1865                 return rte_flow_error_set(error, ENOTSUP,
1866                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1867                                           "specified value not supported");
1868         /* spec and last are valid, validate the specified range. */
1869         fragment_offset_last = last->hdr.fragment_offset &
1870                                mask->hdr.fragment_offset;
1871         /*
1872          * Match on fragment_offset spec 0x2001 and last 0x3fff
1873          * means MF is 1 and frag-offset is > 0.
1874          * This packet is fragment 2nd and onward, excluding last.
1875          * This is not yet supported in MLX5, return appropriate
1876          * error message.
1877          */
1878         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
1879             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1880                 return rte_flow_error_set(error, ENOTSUP,
1881                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1882                                           last, "match on following "
1883                                           "fragments not supported");
1884         /*
1885          * Match on fragment_offset spec 0x0001 and last 0x1fff
1886          * means MF is 0 and frag-offset is > 0.
1887          * This packet is last fragment of fragmented packet.
1888          * This is not yet supported in MLX5, return appropriate
1889          * error message.
1890          */
1891         if (fragment_offset_spec == RTE_BE16(1) &&
1892             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
1893                 return rte_flow_error_set(error, ENOTSUP,
1894                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1895                                           last, "match on last "
1896                                           "fragment not supported");
1897         /*
1898          * Match on fragment_offset spec 0x0001 and last 0x3fff
1899          * means MF and/or frag-offset is not 0.
1900          * This is a fragmented packet.
1901          * Other range values are invalid and rejected.
1902          */
1903         if (!(fragment_offset_spec == RTE_BE16(1) &&
1904               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
1905                 return rte_flow_error_set(error, ENOTSUP,
1906                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
1907                                           "specified range not supported");
1908         return 0;
1909 }
1910
1911 /**
1912  * Validate IPV6 fragment extension item.
1913  *
1914  * @param[in] item
1915  *   Item specification.
1916  * @param[in] item_flags
1917  *   Bit-fields that holds the items detected until now.
1918  * @param[out] error
1919  *   Pointer to error structure.
1920  *
1921  * @return
1922  *   0 on success, a negative errno value otherwise and rte_errno is set.
1923  */
1924 static int
1925 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
1926                                     uint64_t item_flags,
1927                                     struct rte_flow_error *error)
1928 {
1929         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
1930         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
1931         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
1932         rte_be16_t frag_data_spec = 0;
1933         rte_be16_t frag_data_last = 0;
1934         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1935         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1936                                       MLX5_FLOW_LAYER_OUTER_L4;
1937         int ret = 0;
1938         struct rte_flow_item_ipv6_frag_ext nic_mask = {
1939                 .hdr = {
1940                         .next_header = 0xff,
1941                         .frag_data = RTE_BE16(0xffff),
1942                 },
1943         };
1944
1945         if (item_flags & l4m)
1946                 return rte_flow_error_set(error, EINVAL,
1947                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1948                                           "ipv6 fragment extension item cannot "
1949                                           "follow L4 item.");
1950         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
1951             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1954                                           "ipv6 fragment extension item must "
1955                                           "follow ipv6 item");
1956         if (spec && mask)
1957                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
1958         if (!frag_data_spec)
1959                 return 0;
1960         /*
1961          * spec and mask are valid, enforce using full mask to make sure the
1962          * complete value is used correctly.
1963          */
1964         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
1965                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
1966                 return rte_flow_error_set(error, EINVAL,
1967                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1968                                           item, "must use full mask for"
1969                                           " frag_data");
1970         /*
1971          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
1972          * This is 1st fragment of fragmented packet.
1973          */
1974         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
1975                 return rte_flow_error_set(error, ENOTSUP,
1976                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1977                                           "match on first fragment not "
1978                                           "supported");
1979         if (frag_data_spec && !last)
1980                 return rte_flow_error_set(error, EINVAL,
1981                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1982                                           "specified value not supported");
1983         ret = mlx5_flow_item_acceptable
1984                                 (item, (const uint8_t *)mask,
1985                                  (const uint8_t *)&nic_mask,
1986                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
1987                                  MLX5_ITEM_RANGE_ACCEPTED, error);
1988         if (ret)
1989                 return ret;
1990         /* spec and last are valid, validate the specified range. */
1991         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
1992         /*
1993          * Match on frag_data spec 0x0009 and last 0xfff9
1994          * means M is 1 and frag-offset is > 0.
1995          * This packet is fragment 2nd and onward, excluding last.
1996          * This is not yet supported in MLX5, return appropriate
1997          * error message.
1998          */
1999         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2000                                        RTE_IPV6_EHDR_MF_MASK) &&
2001             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2002                 return rte_flow_error_set(error, ENOTSUP,
2003                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2004                                           last, "match on following "
2005                                           "fragments not supported");
2006         /*
2007          * Match on frag_data spec 0x0008 and last 0xfff8
2008          * means M is 0 and frag-offset is > 0.
2009          * This packet is last fragment of fragmented packet.
2010          * This is not yet supported in MLX5, return appropriate
2011          * error message.
2012          */
2013         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2014             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2015                 return rte_flow_error_set(error, ENOTSUP,
2016                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2017                                           last, "match on last "
2018                                           "fragment not supported");
2019         /* Other range values are invalid and rejected. */
2020         return rte_flow_error_set(error, EINVAL,
2021                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2022                                   "specified range not supported");
2023 }
2024
2025 /**
2026  * Validate the pop VLAN action.
2027  *
2028  * @param[in] dev
2029  *   Pointer to the rte_eth_dev structure.
2030  * @param[in] action_flags
2031  *   Holds the actions detected until now.
2032  * @param[in] action
2033  *   Pointer to the pop vlan action.
2034  * @param[in] item_flags
2035  *   The items found in this flow rule.
2036  * @param[in] attr
2037  *   Pointer to flow attributes.
2038  * @param[out] error
2039  *   Pointer to error structure.
2040  *
2041  * @return
2042  *   0 on success, a negative errno value otherwise and rte_errno is set.
2043  */
2044 static int
2045 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2046                                  uint64_t action_flags,
2047                                  const struct rte_flow_action *action,
2048                                  uint64_t item_flags,
2049                                  const struct rte_flow_attr *attr,
2050                                  struct rte_flow_error *error)
2051 {
2052         const struct mlx5_priv *priv = dev->data->dev_private;
2053
2054         (void)action;
2055         (void)attr;
2056         if (!priv->sh->pop_vlan_action)
2057                 return rte_flow_error_set(error, ENOTSUP,
2058                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2059                                           NULL,
2060                                           "pop vlan action is not supported");
2061         if (attr->egress)
2062                 return rte_flow_error_set(error, ENOTSUP,
2063                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2064                                           NULL,
2065                                           "pop vlan action not supported for "
2066                                           "egress");
2067         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2068                 return rte_flow_error_set(error, ENOTSUP,
2069                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2070                                           "no support for multiple VLAN "
2071                                           "actions");
2072         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2073         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2074             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2075                 return rte_flow_error_set(error, ENOTSUP,
2076                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2077                                           NULL,
2078                                           "cannot pop vlan after decap without "
2079                                           "match on inner vlan in the flow");
2080         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2081         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2082             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2083                 return rte_flow_error_set(error, ENOTSUP,
2084                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2085                                           NULL,
2086                                           "cannot pop vlan without a "
2087                                           "match on (outer) vlan in the flow");
2088         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2091                                           "wrong action order, port_id should "
2092                                           "be after pop VLAN action");
2093         if (!attr->transfer && priv->representor)
2094                 return rte_flow_error_set(error, ENOTSUP,
2095                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2096                                           "pop vlan action for VF representor "
2097                                           "not supported on NIC table");
2098         return 0;
2099 }
2100
2101 /**
2102  * Get VLAN default info from vlan match info.
2103  *
2104  * @param[in] items
2105  *   the list of item specifications.
2106  * @param[out] vlan
2107  *   pointer VLAN info to fill to.
2108  *
2109  * @return
2110  *   0 on success, a negative errno value otherwise and rte_errno is set.
2111  */
2112 static void
2113 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2114                                   struct rte_vlan_hdr *vlan)
2115 {
2116         const struct rte_flow_item_vlan nic_mask = {
2117                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2118                                 MLX5DV_FLOW_VLAN_VID_MASK),
2119                 .inner_type = RTE_BE16(0xffff),
2120         };
2121
2122         if (items == NULL)
2123                 return;
2124         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2125                 int type = items->type;
2126
2127                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2128                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2129                         break;
2130         }
2131         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2132                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2133                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2134
2135                 /* If VLAN item in pattern doesn't contain data, return here. */
2136                 if (!vlan_v)
2137                         return;
2138                 if (!vlan_m)
2139                         vlan_m = &nic_mask;
2140                 /* Only full match values are accepted */
2141                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2142                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2143                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2144                         vlan->vlan_tci |=
2145                                 rte_be_to_cpu_16(vlan_v->tci &
2146                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2147                 }
2148                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2149                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2150                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2151                         vlan->vlan_tci |=
2152                                 rte_be_to_cpu_16(vlan_v->tci &
2153                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2154                 }
2155                 if (vlan_m->inner_type == nic_mask.inner_type)
2156                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2157                                                            vlan_m->inner_type);
2158         }
2159 }
2160
2161 /**
2162  * Validate the push VLAN action.
2163  *
2164  * @param[in] dev
2165  *   Pointer to the rte_eth_dev structure.
2166  * @param[in] action_flags
2167  *   Holds the actions detected until now.
2168  * @param[in] item_flags
2169  *   The items found in this flow rule.
2170  * @param[in] action
2171  *   Pointer to the action structure.
2172  * @param[in] attr
2173  *   Pointer to flow attributes
2174  * @param[out] error
2175  *   Pointer to error structure.
2176  *
2177  * @return
2178  *   0 on success, a negative errno value otherwise and rte_errno is set.
2179  */
2180 static int
2181 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2182                                   uint64_t action_flags,
2183                                   const struct rte_flow_item_vlan *vlan_m,
2184                                   const struct rte_flow_action *action,
2185                                   const struct rte_flow_attr *attr,
2186                                   struct rte_flow_error *error)
2187 {
2188         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2189         const struct mlx5_priv *priv = dev->data->dev_private;
2190
2191         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2192             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2193                 return rte_flow_error_set(error, EINVAL,
2194                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2195                                           "invalid vlan ethertype");
2196         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2197                 return rte_flow_error_set(error, EINVAL,
2198                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2199                                           "wrong action order, port_id should "
2200                                           "be after push VLAN");
2201         if (!attr->transfer && priv->representor)
2202                 return rte_flow_error_set(error, ENOTSUP,
2203                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2204                                           "push vlan action for VF representor "
2205                                           "not supported on NIC table");
2206         if (vlan_m &&
2207             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2208             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2209                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2210             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2211             !(mlx5_flow_find_action
2212                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2213                 return rte_flow_error_set(error, EINVAL,
2214                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2215                                           "not full match mask on VLAN PCP and "
2216                                           "there is no of_set_vlan_pcp action, "
2217                                           "push VLAN action cannot figure out "
2218                                           "PCP value");
2219         if (vlan_m &&
2220             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2221             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2222                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2223             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2224             !(mlx5_flow_find_action
2225                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2226                 return rte_flow_error_set(error, EINVAL,
2227                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2228                                           "not full match mask on VLAN VID and "
2229                                           "there is no of_set_vlan_vid action, "
2230                                           "push VLAN action cannot figure out "
2231                                           "VID value");
2232         (void)attr;
2233         return 0;
2234 }
2235
2236 /**
2237  * Validate the set VLAN PCP.
2238  *
2239  * @param[in] action_flags
2240  *   Holds the actions detected until now.
2241  * @param[in] actions
2242  *   Pointer to the list of actions remaining in the flow rule.
2243  * @param[out] error
2244  *   Pointer to error structure.
2245  *
2246  * @return
2247  *   0 on success, a negative errno value otherwise and rte_errno is set.
2248  */
2249 static int
2250 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2251                                      const struct rte_flow_action actions[],
2252                                      struct rte_flow_error *error)
2253 {
2254         const struct rte_flow_action *action = actions;
2255         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2256
2257         if (conf->vlan_pcp > 7)
2258                 return rte_flow_error_set(error, EINVAL,
2259                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2260                                           "VLAN PCP value is too big");
2261         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2262                 return rte_flow_error_set(error, ENOTSUP,
2263                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2264                                           "set VLAN PCP action must follow "
2265                                           "the push VLAN action");
2266         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2267                 return rte_flow_error_set(error, ENOTSUP,
2268                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2269                                           "Multiple VLAN PCP modification are "
2270                                           "not supported");
2271         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2272                 return rte_flow_error_set(error, EINVAL,
2273                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2274                                           "wrong action order, port_id should "
2275                                           "be after set VLAN PCP");
2276         return 0;
2277 }
2278
2279 /**
2280  * Validate the set VLAN VID.
2281  *
2282  * @param[in] item_flags
2283  *   Holds the items detected in this rule.
2284  * @param[in] action_flags
2285  *   Holds the actions detected until now.
2286  * @param[in] actions
2287  *   Pointer to the list of actions remaining in the flow rule.
2288  * @param[out] error
2289  *   Pointer to error structure.
2290  *
2291  * @return
2292  *   0 on success, a negative errno value otherwise and rte_errno is set.
2293  */
2294 static int
2295 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2296                                      uint64_t action_flags,
2297                                      const struct rte_flow_action actions[],
2298                                      struct rte_flow_error *error)
2299 {
2300         const struct rte_flow_action *action = actions;
2301         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2302
2303         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2304                 return rte_flow_error_set(error, EINVAL,
2305                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2306                                           "VLAN VID value is too big");
2307         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2308             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2309                 return rte_flow_error_set(error, ENOTSUP,
2310                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2311                                           "set VLAN VID action must follow push"
2312                                           " VLAN action or match on VLAN item");
2313         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2314                 return rte_flow_error_set(error, ENOTSUP,
2315                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2316                                           "Multiple VLAN VID modifications are "
2317                                           "not supported");
2318         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2319                 return rte_flow_error_set(error, EINVAL,
2320                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2321                                           "wrong action order, port_id should "
2322                                           "be after set VLAN VID");
2323         return 0;
2324 }
2325
2326 /*
2327  * Validate the FLAG action.
2328  *
2329  * @param[in] dev
2330  *   Pointer to the rte_eth_dev structure.
2331  * @param[in] action_flags
2332  *   Holds the actions detected until now.
2333  * @param[in] attr
2334  *   Pointer to flow attributes
2335  * @param[out] error
2336  *   Pointer to error structure.
2337  *
2338  * @return
2339  *   0 on success, a negative errno value otherwise and rte_errno is set.
2340  */
2341 static int
2342 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2343                              uint64_t action_flags,
2344                              const struct rte_flow_attr *attr,
2345                              struct rte_flow_error *error)
2346 {
2347         struct mlx5_priv *priv = dev->data->dev_private;
2348         struct mlx5_dev_config *config = &priv->config;
2349         int ret;
2350
2351         /* Fall back if no extended metadata register support. */
2352         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2353                 return mlx5_flow_validate_action_flag(action_flags, attr,
2354                                                       error);
2355         /* Extensive metadata mode requires registers. */
2356         if (!mlx5_flow_ext_mreg_supported(dev))
2357                 return rte_flow_error_set(error, ENOTSUP,
2358                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2359                                           "no metadata registers "
2360                                           "to support flag action");
2361         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2362                 return rte_flow_error_set(error, ENOTSUP,
2363                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2364                                           "extended metadata register"
2365                                           " isn't available");
2366         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2367         if (ret < 0)
2368                 return ret;
2369         MLX5_ASSERT(ret > 0);
2370         if (action_flags & MLX5_FLOW_ACTION_MARK)
2371                 return rte_flow_error_set(error, EINVAL,
2372                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2373                                           "can't mark and flag in same flow");
2374         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2375                 return rte_flow_error_set(error, EINVAL,
2376                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2377                                           "can't have 2 flag"
2378                                           " actions in same flow");
2379         return 0;
2380 }
2381
2382 /**
2383  * Validate MARK action.
2384  *
2385  * @param[in] dev
2386  *   Pointer to the rte_eth_dev structure.
2387  * @param[in] action
2388  *   Pointer to action.
2389  * @param[in] action_flags
2390  *   Holds the actions detected until now.
2391  * @param[in] attr
2392  *   Pointer to flow attributes
2393  * @param[out] error
2394  *   Pointer to error structure.
2395  *
2396  * @return
2397  *   0 on success, a negative errno value otherwise and rte_errno is set.
2398  */
2399 static int
2400 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2401                              const struct rte_flow_action *action,
2402                              uint64_t action_flags,
2403                              const struct rte_flow_attr *attr,
2404                              struct rte_flow_error *error)
2405 {
2406         struct mlx5_priv *priv = dev->data->dev_private;
2407         struct mlx5_dev_config *config = &priv->config;
2408         const struct rte_flow_action_mark *mark = action->conf;
2409         int ret;
2410
2411         /* Fall back if no extended metadata register support. */
2412         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2413                 return mlx5_flow_validate_action_mark(action, action_flags,
2414                                                       attr, error);
2415         /* Extensive metadata mode requires registers. */
2416         if (!mlx5_flow_ext_mreg_supported(dev))
2417                 return rte_flow_error_set(error, ENOTSUP,
2418                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2419                                           "no metadata registers "
2420                                           "to support mark action");
2421         if (!priv->sh->dv_mark_mask)
2422                 return rte_flow_error_set(error, ENOTSUP,
2423                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2424                                           "extended metadata register"
2425                                           " isn't available");
2426         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2427         if (ret < 0)
2428                 return ret;
2429         MLX5_ASSERT(ret > 0);
2430         if (!mark)
2431                 return rte_flow_error_set(error, EINVAL,
2432                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2433                                           "configuration cannot be null");
2434         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2435                 return rte_flow_error_set(error, EINVAL,
2436                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2437                                           &mark->id,
2438                                           "mark id exceeds the limit");
2439         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2440                 return rte_flow_error_set(error, EINVAL,
2441                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2442                                           "can't flag and mark in same flow");
2443         if (action_flags & MLX5_FLOW_ACTION_MARK)
2444                 return rte_flow_error_set(error, EINVAL,
2445                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2446                                           "can't have 2 mark actions in same"
2447                                           " flow");
2448         return 0;
2449 }
2450
2451 /**
2452  * Validate SET_META action.
2453  *
2454  * @param[in] dev
2455  *   Pointer to the rte_eth_dev structure.
2456  * @param[in] action
2457  *   Pointer to the action structure.
2458  * @param[in] action_flags
2459  *   Holds the actions detected until now.
2460  * @param[in] attr
2461  *   Pointer to flow attributes
2462  * @param[out] error
2463  *   Pointer to error structure.
2464  *
2465  * @return
2466  *   0 on success, a negative errno value otherwise and rte_errno is set.
2467  */
2468 static int
2469 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2470                                  const struct rte_flow_action *action,
2471                                  uint64_t action_flags __rte_unused,
2472                                  const struct rte_flow_attr *attr,
2473                                  struct rte_flow_error *error)
2474 {
2475         const struct rte_flow_action_set_meta *conf;
2476         uint32_t nic_mask = UINT32_MAX;
2477         int reg;
2478
2479         if (!mlx5_flow_ext_mreg_supported(dev))
2480                 return rte_flow_error_set(error, ENOTSUP,
2481                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2482                                           "extended metadata register"
2483                                           " isn't supported");
2484         reg = flow_dv_get_metadata_reg(dev, attr, error);
2485         if (reg < 0)
2486                 return reg;
2487         if (reg != REG_A && reg != REG_B) {
2488                 struct mlx5_priv *priv = dev->data->dev_private;
2489
2490                 nic_mask = priv->sh->dv_meta_mask;
2491         }
2492         if (!(action->conf))
2493                 return rte_flow_error_set(error, EINVAL,
2494                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2495                                           "configuration cannot be null");
2496         conf = (const struct rte_flow_action_set_meta *)action->conf;
2497         if (!conf->mask)
2498                 return rte_flow_error_set(error, EINVAL,
2499                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2500                                           "zero mask doesn't have any effect");
2501         if (conf->mask & ~nic_mask)
2502                 return rte_flow_error_set(error, EINVAL,
2503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2504                                           "meta data must be within reg C0");
2505         return 0;
2506 }
2507
2508 /**
2509  * Validate SET_TAG action.
2510  *
2511  * @param[in] dev
2512  *   Pointer to the rte_eth_dev structure.
2513  * @param[in] action
2514  *   Pointer to the action structure.
2515  * @param[in] action_flags
2516  *   Holds the actions detected until now.
2517  * @param[in] attr
2518  *   Pointer to flow attributes
2519  * @param[out] error
2520  *   Pointer to error structure.
2521  *
2522  * @return
2523  *   0 on success, a negative errno value otherwise and rte_errno is set.
2524  */
2525 static int
2526 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2527                                 const struct rte_flow_action *action,
2528                                 uint64_t action_flags,
2529                                 const struct rte_flow_attr *attr,
2530                                 struct rte_flow_error *error)
2531 {
2532         const struct rte_flow_action_set_tag *conf;
2533         const uint64_t terminal_action_flags =
2534                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2535                 MLX5_FLOW_ACTION_RSS;
2536         int ret;
2537
2538         if (!mlx5_flow_ext_mreg_supported(dev))
2539                 return rte_flow_error_set(error, ENOTSUP,
2540                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2541                                           "extensive metadata register"
2542                                           " isn't supported");
2543         if (!(action->conf))
2544                 return rte_flow_error_set(error, EINVAL,
2545                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2546                                           "configuration cannot be null");
2547         conf = (const struct rte_flow_action_set_tag *)action->conf;
2548         if (!conf->mask)
2549                 return rte_flow_error_set(error, EINVAL,
2550                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2551                                           "zero mask doesn't have any effect");
2552         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2553         if (ret < 0)
2554                 return ret;
2555         if (!attr->transfer && attr->ingress &&
2556             (action_flags & terminal_action_flags))
2557                 return rte_flow_error_set(error, EINVAL,
2558                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2559                                           "set_tag has no effect"
2560                                           " with terminal actions");
2561         return 0;
2562 }
2563
2564 /**
2565  * Validate count action.
2566  *
2567  * @param[in] dev
2568  *   Pointer to rte_eth_dev structure.
2569  * @param[out] error
2570  *   Pointer to error structure.
2571  *
2572  * @return
2573  *   0 on success, a negative errno value otherwise and rte_errno is set.
2574  */
2575 static int
2576 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2577                               struct rte_flow_error *error)
2578 {
2579         struct mlx5_priv *priv = dev->data->dev_private;
2580
2581         if (!priv->config.devx)
2582                 goto notsup_err;
2583 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2584         return 0;
2585 #endif
2586 notsup_err:
2587         return rte_flow_error_set
2588                       (error, ENOTSUP,
2589                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2590                        NULL,
2591                        "count action not supported");
2592 }
2593
2594 /**
2595  * Validate the L2 encap action.
2596  *
2597  * @param[in] dev
2598  *   Pointer to the rte_eth_dev structure.
2599  * @param[in] action_flags
2600  *   Holds the actions detected until now.
2601  * @param[in] action
2602  *   Pointer to the action structure.
2603  * @param[in] attr
2604  *   Pointer to flow attributes.
2605  * @param[out] error
2606  *   Pointer to error structure.
2607  *
2608  * @return
2609  *   0 on success, a negative errno value otherwise and rte_errno is set.
2610  */
2611 static int
2612 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2613                                  uint64_t action_flags,
2614                                  const struct rte_flow_action *action,
2615                                  const struct rte_flow_attr *attr,
2616                                  struct rte_flow_error *error)
2617 {
2618         const struct mlx5_priv *priv = dev->data->dev_private;
2619
2620         if (!(action->conf))
2621                 return rte_flow_error_set(error, EINVAL,
2622                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2623                                           "configuration cannot be null");
2624         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2625                 return rte_flow_error_set(error, EINVAL,
2626                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2627                                           "can only have a single encap action "
2628                                           "in a flow");
2629         if (!attr->transfer && priv->representor)
2630                 return rte_flow_error_set(error, ENOTSUP,
2631                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2632                                           "encap action for VF representor "
2633                                           "not supported on NIC table");
2634         return 0;
2635 }
2636
2637 /**
2638  * Validate a decap action.
2639  *
2640  * @param[in] dev
2641  *   Pointer to the rte_eth_dev structure.
2642  * @param[in] action_flags
2643  *   Holds the actions detected until now.
2644  * @param[in] attr
2645  *   Pointer to flow attributes
2646  * @param[out] error
2647  *   Pointer to error structure.
2648  *
2649  * @return
2650  *   0 on success, a negative errno value otherwise and rte_errno is set.
2651  */
2652 static int
2653 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2654                               uint64_t action_flags,
2655                               const struct rte_flow_attr *attr,
2656                               struct rte_flow_error *error)
2657 {
2658         const struct mlx5_priv *priv = dev->data->dev_private;
2659
2660         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2661             !priv->config.decap_en)
2662                 return rte_flow_error_set(error, ENOTSUP,
2663                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2664                                           "decap is not enabled");
2665         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2666                 return rte_flow_error_set(error, ENOTSUP,
2667                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2668                                           action_flags &
2669                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2670                                           "have a single decap action" : "decap "
2671                                           "after encap is not supported");
2672         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2673                 return rte_flow_error_set(error, EINVAL,
2674                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2675                                           "can't have decap action after"
2676                                           " modify action");
2677         if (attr->egress)
2678                 return rte_flow_error_set(error, ENOTSUP,
2679                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2680                                           NULL,
2681                                           "decap action not supported for "
2682                                           "egress");
2683         if (!attr->transfer && priv->representor)
2684                 return rte_flow_error_set(error, ENOTSUP,
2685                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2686                                           "decap action for VF representor "
2687                                           "not supported on NIC table");
2688         return 0;
2689 }
2690
2691 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2692
2693 /**
2694  * Validate the raw encap and decap actions.
2695  *
2696  * @param[in] dev
2697  *   Pointer to the rte_eth_dev structure.
2698  * @param[in] decap
2699  *   Pointer to the decap action.
2700  * @param[in] encap
2701  *   Pointer to the encap action.
2702  * @param[in] attr
2703  *   Pointer to flow attributes
2704  * @param[in/out] action_flags
2705  *   Holds the actions detected until now.
2706  * @param[out] actions_n
2707  *   pointer to the number of actions counter.
2708  * @param[out] error
2709  *   Pointer to error structure.
2710  *
2711  * @return
2712  *   0 on success, a negative errno value otherwise and rte_errno is set.
2713  */
2714 static int
2715 flow_dv_validate_action_raw_encap_decap
2716         (struct rte_eth_dev *dev,
2717          const struct rte_flow_action_raw_decap *decap,
2718          const struct rte_flow_action_raw_encap *encap,
2719          const struct rte_flow_attr *attr, uint64_t *action_flags,
2720          int *actions_n, struct rte_flow_error *error)
2721 {
2722         const struct mlx5_priv *priv = dev->data->dev_private;
2723         int ret;
2724
2725         if (encap && (!encap->size || !encap->data))
2726                 return rte_flow_error_set(error, EINVAL,
2727                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2728                                           "raw encap data cannot be empty");
2729         if (decap && encap) {
2730                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2731                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2732                         /* L3 encap. */
2733                         decap = NULL;
2734                 else if (encap->size <=
2735                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2736                            decap->size >
2737                            MLX5_ENCAPSULATION_DECISION_SIZE)
2738                         /* L3 decap. */
2739                         encap = NULL;
2740                 else if (encap->size >
2741                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2742                            decap->size >
2743                            MLX5_ENCAPSULATION_DECISION_SIZE)
2744                         /* 2 L2 actions: encap and decap. */
2745                         ;
2746                 else
2747                         return rte_flow_error_set(error,
2748                                 ENOTSUP,
2749                                 RTE_FLOW_ERROR_TYPE_ACTION,
2750                                 NULL, "unsupported too small "
2751                                 "raw decap and too small raw "
2752                                 "encap combination");
2753         }
2754         if (decap) {
2755                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2756                                                     error);
2757                 if (ret < 0)
2758                         return ret;
2759                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2760                 ++(*actions_n);
2761         }
2762         if (encap) {
2763                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2764                         return rte_flow_error_set(error, ENOTSUP,
2765                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2766                                                   NULL,
2767                                                   "small raw encap size");
2768                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2769                         return rte_flow_error_set(error, EINVAL,
2770                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2771                                                   NULL,
2772                                                   "more than one encap action");
2773                 if (!attr->transfer && priv->representor)
2774                         return rte_flow_error_set
2775                                         (error, ENOTSUP,
2776                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2777                                          "encap action for VF representor "
2778                                          "not supported on NIC table");
2779                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2780                 ++(*actions_n);
2781         }
2782         return 0;
2783 }
2784
2785 /**
2786  * Match encap_decap resource.
2787  *
2788  * @param entry
2789  *   Pointer to exist resource entry object.
2790  * @param ctx
2791  *   Pointer to new encap_decap resource.
2792  *
2793  * @return
2794  *   0 on matching, -1 otherwise.
2795  */
2796 static int
2797 flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
2798 {
2799         struct mlx5_flow_dv_encap_decap_resource *resource;
2800         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2801
2802         resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
2803         cache_resource = container_of(entry,
2804                                       struct mlx5_flow_dv_encap_decap_resource,
2805                                       entry);
2806         if (resource->entry.key == cache_resource->entry.key &&
2807             resource->reformat_type == cache_resource->reformat_type &&
2808             resource->ft_type == cache_resource->ft_type &&
2809             resource->flags == cache_resource->flags &&
2810             resource->size == cache_resource->size &&
2811             !memcmp((const void *)resource->buf,
2812                     (const void *)cache_resource->buf,
2813                     resource->size))
2814                 return 0;
2815         return -1;
2816 }
2817
2818 /**
2819  * Find existing encap/decap resource or create and register a new one.
2820  *
2821  * @param[in, out] dev
2822  *   Pointer to rte_eth_dev structure.
2823  * @param[in, out] resource
2824  *   Pointer to encap/decap resource.
2825  * @parm[in, out] dev_flow
2826  *   Pointer to the dev_flow.
2827  * @param[out] error
2828  *   pointer to error structure.
2829  *
2830  * @return
2831  *   0 on success otherwise -errno and errno is set.
2832  */
2833 static int
2834 flow_dv_encap_decap_resource_register
2835                         (struct rte_eth_dev *dev,
2836                          struct mlx5_flow_dv_encap_decap_resource *resource,
2837                          struct mlx5_flow *dev_flow,
2838                          struct rte_flow_error *error)
2839 {
2840         struct mlx5_priv *priv = dev->data->dev_private;
2841         struct mlx5_dev_ctx_shared *sh = priv->sh;
2842         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2843         struct mlx5dv_dr_domain *domain;
2844         struct mlx5_hlist_entry *entry;
2845         union mlx5_flow_encap_decap_key encap_decap_key = {
2846                 {
2847                         .ft_type = resource->ft_type,
2848                         .refmt_type = resource->reformat_type,
2849                         .buf_size = resource->size,
2850                         .table_level = !!dev_flow->dv.group,
2851                         .cksum = 0,
2852                 }
2853         };
2854         int ret;
2855
2856         resource->flags = dev_flow->dv.group ? 0 : 1;
2857         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2858                 domain = sh->fdb_domain;
2859         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2860                 domain = sh->rx_domain;
2861         else
2862                 domain = sh->tx_domain;
2863         encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
2864                                                 resource->size, 0);
2865         resource->entry.key = encap_decap_key.v64;
2866         /* Lookup a matching resource from cache. */
2867         entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
2868                                      flow_dv_encap_decap_resource_match,
2869                                      (void *)resource);
2870         if (entry) {
2871                 cache_resource = container_of(entry,
2872                         struct mlx5_flow_dv_encap_decap_resource, entry);
2873                 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2874                         (void *)cache_resource,
2875                         rte_atomic32_read(&cache_resource->refcnt));
2876                 rte_atomic32_inc(&cache_resource->refcnt);
2877                 dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
2878                 dev_flow->dv.encap_decap = cache_resource;
2879                 return 0;
2880         }
2881         /* Register new encap/decap resource. */
2882         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2883                                        &dev_flow->handle->dvh.rix_encap_decap);
2884         if (!cache_resource)
2885                 return rte_flow_error_set(error, ENOMEM,
2886                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2887                                           "cannot allocate resource memory");
2888         *cache_resource = *resource;
2889         cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
2890         ret = mlx5_flow_os_create_flow_action_packet_reformat
2891                                         (sh->ctx, domain, cache_resource,
2892                                          &cache_resource->action);
2893         if (ret) {
2894                 mlx5_free(cache_resource);
2895                 return rte_flow_error_set(error, ENOMEM,
2896                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2897                                           NULL, "cannot create action");
2898         }
2899         rte_atomic32_init(&cache_resource->refcnt);
2900         rte_atomic32_inc(&cache_resource->refcnt);
2901         if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
2902                                  flow_dv_encap_decap_resource_match,
2903                                  (void *)cache_resource)) {
2904                 claim_zero(mlx5_flow_os_destroy_flow_action
2905                                                 (cache_resource->action));
2906                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2907                                 cache_resource->idx);
2908                 return rte_flow_error_set(error, EEXIST,
2909                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2910                                           NULL, "action exist");
2911         }
2912         dev_flow->dv.encap_decap = cache_resource;
2913         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2914                 (void *)cache_resource,
2915                 rte_atomic32_read(&cache_resource->refcnt));
2916         return 0;
2917 }
2918
2919 /**
2920  * Find existing table jump resource or create and register a new one.
2921  *
2922  * @param[in, out] dev
2923  *   Pointer to rte_eth_dev structure.
2924  * @param[in, out] tbl
2925  *   Pointer to flow table resource.
2926  * @parm[in, out] dev_flow
2927  *   Pointer to the dev_flow.
2928  * @param[out] error
2929  *   pointer to error structure.
2930  *
2931  * @return
2932  *   0 on success otherwise -errno and errno is set.
2933  */
2934 static int
2935 flow_dv_jump_tbl_resource_register
2936                         (struct rte_eth_dev *dev __rte_unused,
2937                          struct mlx5_flow_tbl_resource *tbl,
2938                          struct mlx5_flow *dev_flow,
2939                          struct rte_flow_error *error)
2940 {
2941         struct mlx5_flow_tbl_data_entry *tbl_data =
2942                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2943         int cnt, ret;
2944
2945         MLX5_ASSERT(tbl);
2946         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2947         if (!cnt) {
2948                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2949                                 (tbl->obj, &tbl_data->jump.action);
2950                 if (ret)
2951                         return rte_flow_error_set(error, ENOMEM,
2952                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2953                                         NULL, "cannot create jump action");
2954                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2955                         (void *)&tbl_data->jump, cnt);
2956         } else {
2957                 /* old jump should not make the table ref++. */
2958                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2959                 MLX5_ASSERT(tbl_data->jump.action);
2960                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2961                         (void *)&tbl_data->jump, cnt);
2962         }
2963         rte_atomic32_inc(&tbl_data->jump.refcnt);
2964         dev_flow->handle->rix_jump = tbl_data->idx;
2965         dev_flow->dv.jump = &tbl_data->jump;
2966         return 0;
2967 }
2968
2969 /**
2970  * Find existing default miss resource or create and register a new one.
2971  *
2972  * @param[in, out] dev
2973  *   Pointer to rte_eth_dev structure.
2974  * @param[out] error
2975  *   pointer to error structure.
2976  *
2977  * @return
2978  *   0 on success otherwise -errno and errno is set.
2979  */
2980 static int
2981 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2982                 struct rte_flow_error *error)
2983 {
2984         struct mlx5_priv *priv = dev->data->dev_private;
2985         struct mlx5_dev_ctx_shared *sh = priv->sh;
2986         struct mlx5_flow_default_miss_resource *cache_resource =
2987                         &sh->default_miss;
2988         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2989
2990         if (!cnt) {
2991                 MLX5_ASSERT(cache_resource->action);
2992                 cache_resource->action =
2993                 mlx5_glue->dr_create_flow_action_default_miss();
2994                 if (!cache_resource->action)
2995                         return rte_flow_error_set(error, ENOMEM,
2996                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2997                                         "cannot create default miss action");
2998                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2999                                 (void *)cache_resource->action, cnt);
3000         }
3001         rte_atomic32_inc(&cache_resource->refcnt);
3002         return 0;
3003 }
3004
3005 /**
3006  * Find existing table port ID resource or create and register a new one.
3007  *
3008  * @param[in, out] dev
3009  *   Pointer to rte_eth_dev structure.
3010  * @param[in, out] resource
3011  *   Pointer to port ID action resource.
3012  * @parm[in, out] dev_flow
3013  *   Pointer to the dev_flow.
3014  * @param[out] error
3015  *   pointer to error structure.
3016  *
3017  * @return
3018  *   0 on success otherwise -errno and errno is set.
3019  */
3020 static int
3021 flow_dv_port_id_action_resource_register
3022                         (struct rte_eth_dev *dev,
3023                          struct mlx5_flow_dv_port_id_action_resource *resource,
3024                          struct mlx5_flow *dev_flow,
3025                          struct rte_flow_error *error)
3026 {
3027         struct mlx5_priv *priv = dev->data->dev_private;
3028         struct mlx5_dev_ctx_shared *sh = priv->sh;
3029         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
3030         uint32_t idx = 0;
3031         int ret;
3032
3033         /* Lookup a matching resource from cache. */
3034         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
3035                       idx, cache_resource, next) {
3036                 if (resource->port_id == cache_resource->port_id) {
3037                         DRV_LOG(DEBUG, "port id action resource resource %p: "
3038                                 "refcnt %d++",
3039                                 (void *)cache_resource,
3040                                 rte_atomic32_read(&cache_resource->refcnt));
3041                         rte_atomic32_inc(&cache_resource->refcnt);
3042                         dev_flow->handle->rix_port_id_action = idx;
3043                         dev_flow->dv.port_id_action = cache_resource;
3044                         return 0;
3045                 }
3046         }
3047         /* Register new port id action resource. */
3048         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
3049                                        &dev_flow->handle->rix_port_id_action);
3050         if (!cache_resource)
3051                 return rte_flow_error_set(error, ENOMEM,
3052                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3053                                           "cannot allocate resource memory");
3054         *cache_resource = *resource;
3055         ret = mlx5_flow_os_create_flow_action_dest_port
3056                                 (priv->sh->fdb_domain, resource->port_id,
3057                                  &cache_resource->action);
3058         if (ret) {
3059                 mlx5_free(cache_resource);
3060                 return rte_flow_error_set(error, ENOMEM,
3061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3062                                           NULL, "cannot create action");
3063         }
3064         rte_atomic32_init(&cache_resource->refcnt);
3065         rte_atomic32_inc(&cache_resource->refcnt);
3066         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
3067                      dev_flow->handle->rix_port_id_action, cache_resource,
3068                      next);
3069         dev_flow->dv.port_id_action = cache_resource;
3070         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
3071                 (void *)cache_resource,
3072                 rte_atomic32_read(&cache_resource->refcnt));
3073         return 0;
3074 }
3075
3076 /**
3077  * Find existing push vlan resource or create and register a new one.
3078  *
3079  * @param [in, out] dev
3080  *   Pointer to rte_eth_dev structure.
3081  * @param[in, out] resource
3082  *   Pointer to port ID action resource.
3083  * @parm[in, out] dev_flow
3084  *   Pointer to the dev_flow.
3085  * @param[out] error
3086  *   pointer to error structure.
3087  *
3088  * @return
3089  *   0 on success otherwise -errno and errno is set.
3090  */
3091 static int
3092 flow_dv_push_vlan_action_resource_register
3093                        (struct rte_eth_dev *dev,
3094                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
3095                         struct mlx5_flow *dev_flow,
3096                         struct rte_flow_error *error)
3097 {
3098         struct mlx5_priv *priv = dev->data->dev_private;
3099         struct mlx5_dev_ctx_shared *sh = priv->sh;
3100         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
3101         struct mlx5dv_dr_domain *domain;
3102         uint32_t idx = 0;
3103         int ret;
3104
3105         /* Lookup a matching resource from cache. */
3106         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3107                       sh->push_vlan_action_list, idx, cache_resource, next) {
3108                 if (resource->vlan_tag == cache_resource->vlan_tag &&
3109                     resource->ft_type == cache_resource->ft_type) {
3110                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
3111                                 "refcnt %d++",
3112                                 (void *)cache_resource,
3113                                 rte_atomic32_read(&cache_resource->refcnt));
3114                         rte_atomic32_inc(&cache_resource->refcnt);
3115                         dev_flow->handle->dvh.rix_push_vlan = idx;
3116                         dev_flow->dv.push_vlan_res = cache_resource;
3117                         return 0;
3118                 }
3119         }
3120         /* Register new push_vlan action resource. */
3121         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3122                                        &dev_flow->handle->dvh.rix_push_vlan);
3123         if (!cache_resource)
3124                 return rte_flow_error_set(error, ENOMEM,
3125                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3126                                           "cannot allocate resource memory");
3127         *cache_resource = *resource;
3128         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3129                 domain = sh->fdb_domain;
3130         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3131                 domain = sh->rx_domain;
3132         else
3133                 domain = sh->tx_domain;
3134         ret = mlx5_flow_os_create_flow_action_push_vlan
3135                                         (domain, resource->vlan_tag,
3136                                          &cache_resource->action);
3137         if (ret) {
3138                 mlx5_free(cache_resource);
3139                 return rte_flow_error_set(error, ENOMEM,
3140                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3141                                           NULL, "cannot create action");
3142         }
3143         rte_atomic32_init(&cache_resource->refcnt);
3144         rte_atomic32_inc(&cache_resource->refcnt);
3145         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3146                      &sh->push_vlan_action_list,
3147                      dev_flow->handle->dvh.rix_push_vlan,
3148                      cache_resource, next);
3149         dev_flow->dv.push_vlan_res = cache_resource;
3150         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
3151                 (void *)cache_resource,
3152                 rte_atomic32_read(&cache_resource->refcnt));
3153         return 0;
3154 }
3155 /**
3156  * Get the size of specific rte_flow_item_type hdr size
3157  *
3158  * @param[in] item_type
3159  *   Tested rte_flow_item_type.
3160  *
3161  * @return
3162  *   sizeof struct item_type, 0 if void or irrelevant.
3163  */
3164 static size_t
3165 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3166 {
3167         size_t retval;
3168
3169         switch (item_type) {
3170         case RTE_FLOW_ITEM_TYPE_ETH:
3171                 retval = sizeof(struct rte_ether_hdr);
3172                 break;
3173         case RTE_FLOW_ITEM_TYPE_VLAN:
3174                 retval = sizeof(struct rte_vlan_hdr);
3175                 break;
3176         case RTE_FLOW_ITEM_TYPE_IPV4:
3177                 retval = sizeof(struct rte_ipv4_hdr);
3178                 break;
3179         case RTE_FLOW_ITEM_TYPE_IPV6:
3180                 retval = sizeof(struct rte_ipv6_hdr);
3181                 break;
3182         case RTE_FLOW_ITEM_TYPE_UDP:
3183                 retval = sizeof(struct rte_udp_hdr);
3184                 break;
3185         case RTE_FLOW_ITEM_TYPE_TCP:
3186                 retval = sizeof(struct rte_tcp_hdr);
3187                 break;
3188         case RTE_FLOW_ITEM_TYPE_VXLAN:
3189         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3190                 retval = sizeof(struct rte_vxlan_hdr);
3191                 break;
3192         case RTE_FLOW_ITEM_TYPE_GRE:
3193         case RTE_FLOW_ITEM_TYPE_NVGRE:
3194                 retval = sizeof(struct rte_gre_hdr);
3195                 break;
3196         case RTE_FLOW_ITEM_TYPE_MPLS:
3197                 retval = sizeof(struct rte_mpls_hdr);
3198                 break;
3199         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
3200         default:
3201                 retval = 0;
3202                 break;
3203         }
3204         return retval;
3205 }
3206
3207 #define MLX5_ENCAP_IPV4_VERSION         0x40
3208 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
3209 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
3210 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
3211 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
3212 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
3213 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
3214
3215 /**
3216  * Convert the encap action data from list of rte_flow_item to raw buffer
3217  *
3218  * @param[in] items
3219  *   Pointer to rte_flow_item objects list.
3220  * @param[out] buf
3221  *   Pointer to the output buffer.
3222  * @param[out] size
3223  *   Pointer to the output buffer size.
3224  * @param[out] error
3225  *   Pointer to the error structure.
3226  *
3227  * @return
3228  *   0 on success, a negative errno value otherwise and rte_errno is set.
3229  */
3230 static int
3231 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
3232                            size_t *size, struct rte_flow_error *error)
3233 {
3234         struct rte_ether_hdr *eth = NULL;
3235         struct rte_vlan_hdr *vlan = NULL;
3236         struct rte_ipv4_hdr *ipv4 = NULL;
3237         struct rte_ipv6_hdr *ipv6 = NULL;
3238         struct rte_udp_hdr *udp = NULL;
3239         struct rte_vxlan_hdr *vxlan = NULL;
3240         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
3241         struct rte_gre_hdr *gre = NULL;
3242         size_t len;
3243         size_t temp_size = 0;
3244
3245         if (!items)
3246                 return rte_flow_error_set(error, EINVAL,
3247                                           RTE_FLOW_ERROR_TYPE_ACTION,
3248                                           NULL, "invalid empty data");
3249         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3250                 len = flow_dv_get_item_hdr_len(items->type);
3251                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
3252                         return rte_flow_error_set(error, EINVAL,
3253                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3254                                                   (void *)items->type,
3255                                                   "items total size is too big"
3256                                                   " for encap action");
3257                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
3258                 switch (items->type) {
3259                 case RTE_FLOW_ITEM_TYPE_ETH:
3260                         eth = (struct rte_ether_hdr *)&buf[temp_size];
3261                         break;
3262                 case RTE_FLOW_ITEM_TYPE_VLAN:
3263                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
3264                         if (!eth)
3265                                 return rte_flow_error_set(error, EINVAL,
3266                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3267                                                 (void *)items->type,
3268                                                 "eth header not found");
3269                         if (!eth->ether_type)
3270                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
3271                         break;
3272                 case RTE_FLOW_ITEM_TYPE_IPV4:
3273                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3274                         if (!vlan && !eth)
3275                                 return rte_flow_error_set(error, EINVAL,
3276                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3277                                                 (void *)items->type,
3278                                                 "neither eth nor vlan"
3279                                                 " header found");
3280                         if (vlan && !vlan->eth_proto)
3281                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3282                         else if (eth && !eth->ether_type)
3283                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3284                         if (!ipv4->version_ihl)
3285                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3286                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3287                         if (!ipv4->time_to_live)
3288                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3289                         break;
3290                 case RTE_FLOW_ITEM_TYPE_IPV6:
3291                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3292                         if (!vlan && !eth)
3293                                 return rte_flow_error_set(error, EINVAL,
3294                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3295                                                 (void *)items->type,
3296                                                 "neither eth nor vlan"
3297                                                 " header found");
3298                         if (vlan && !vlan->eth_proto)
3299                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3300                         else if (eth && !eth->ether_type)
3301                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3302                         if (!ipv6->vtc_flow)
3303                                 ipv6->vtc_flow =
3304                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3305                         if (!ipv6->hop_limits)
3306                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3307                         break;
3308                 case RTE_FLOW_ITEM_TYPE_UDP:
3309                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3310                         if (!ipv4 && !ipv6)
3311                                 return rte_flow_error_set(error, EINVAL,
3312                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3313                                                 (void *)items->type,
3314                                                 "ip header not found");
3315                         if (ipv4 && !ipv4->next_proto_id)
3316                                 ipv4->next_proto_id = IPPROTO_UDP;
3317                         else if (ipv6 && !ipv6->proto)
3318                                 ipv6->proto = IPPROTO_UDP;
3319                         break;
3320                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3321                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3322                         if (!udp)
3323                                 return rte_flow_error_set(error, EINVAL,
3324                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3325                                                 (void *)items->type,
3326                                                 "udp header not found");
3327                         if (!udp->dst_port)
3328                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3329                         if (!vxlan->vx_flags)
3330                                 vxlan->vx_flags =
3331                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3332                         break;
3333                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3334                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3335                         if (!udp)
3336                                 return rte_flow_error_set(error, EINVAL,
3337                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3338                                                 (void *)items->type,
3339                                                 "udp header not found");
3340                         if (!vxlan_gpe->proto)
3341                                 return rte_flow_error_set(error, EINVAL,
3342                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3343                                                 (void *)items->type,
3344                                                 "next protocol not found");
3345                         if (!udp->dst_port)
3346                                 udp->dst_port =
3347                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3348                         if (!vxlan_gpe->vx_flags)
3349                                 vxlan_gpe->vx_flags =
3350                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3351                         break;
3352                 case RTE_FLOW_ITEM_TYPE_GRE:
3353                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3354                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3355                         if (!gre->proto)
3356                                 return rte_flow_error_set(error, EINVAL,
3357                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3358                                                 (void *)items->type,
3359                                                 "next protocol not found");
3360                         if (!ipv4 && !ipv6)
3361                                 return rte_flow_error_set(error, EINVAL,
3362                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3363                                                 (void *)items->type,
3364                                                 "ip header not found");
3365                         if (ipv4 && !ipv4->next_proto_id)
3366                                 ipv4->next_proto_id = IPPROTO_GRE;
3367                         else if (ipv6 && !ipv6->proto)
3368                                 ipv6->proto = IPPROTO_GRE;
3369                         break;
3370                 case RTE_FLOW_ITEM_TYPE_VOID:
3371                         break;
3372                 default:
3373                         return rte_flow_error_set(error, EINVAL,
3374                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3375                                                   (void *)items->type,
3376                                                   "unsupported item type");
3377                         break;
3378                 }
3379                 temp_size += len;
3380         }
3381         *size = temp_size;
3382         return 0;
3383 }
3384
3385 static int
3386 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3387 {
3388         struct rte_ether_hdr *eth = NULL;
3389         struct rte_vlan_hdr *vlan = NULL;
3390         struct rte_ipv6_hdr *ipv6 = NULL;
3391         struct rte_udp_hdr *udp = NULL;
3392         char *next_hdr;
3393         uint16_t proto;
3394
3395         eth = (struct rte_ether_hdr *)data;
3396         next_hdr = (char *)(eth + 1);
3397         proto = RTE_BE16(eth->ether_type);
3398
3399         /* VLAN skipping */
3400         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3401                 vlan = (struct rte_vlan_hdr *)next_hdr;
3402                 proto = RTE_BE16(vlan->eth_proto);
3403                 next_hdr += sizeof(struct rte_vlan_hdr);
3404         }
3405
3406         /* HW calculates IPv4 csum. no need to proceed */
3407         if (proto == RTE_ETHER_TYPE_IPV4)
3408                 return 0;
3409
3410         /* non IPv4/IPv6 header. not supported */
3411         if (proto != RTE_ETHER_TYPE_IPV6) {
3412                 return rte_flow_error_set(error, ENOTSUP,
3413                                           RTE_FLOW_ERROR_TYPE_ACTION,
3414                                           NULL, "Cannot offload non IPv4/IPv6");
3415         }
3416
3417         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3418
3419         /* ignore non UDP */
3420         if (ipv6->proto != IPPROTO_UDP)
3421                 return 0;
3422
3423         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3424         udp->dgram_cksum = 0;
3425
3426         return 0;
3427 }
3428
3429 /**
3430  * Convert L2 encap action to DV specification.
3431  *
3432  * @param[in] dev
3433  *   Pointer to rte_eth_dev structure.
3434  * @param[in] action
3435  *   Pointer to action structure.
3436  * @param[in, out] dev_flow
3437  *   Pointer to the mlx5_flow.
3438  * @param[in] transfer
3439  *   Mark if the flow is E-Switch flow.
3440  * @param[out] error
3441  *   Pointer to the error structure.
3442  *
3443  * @return
3444  *   0 on success, a negative errno value otherwise and rte_errno is set.
3445  */
3446 static int
3447 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3448                                const struct rte_flow_action *action,
3449                                struct mlx5_flow *dev_flow,
3450                                uint8_t transfer,
3451                                struct rte_flow_error *error)
3452 {
3453         const struct rte_flow_item *encap_data;
3454         const struct rte_flow_action_raw_encap *raw_encap_data;
3455         struct mlx5_flow_dv_encap_decap_resource res = {
3456                 .reformat_type =
3457                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3458                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3459                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3460         };
3461
3462         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3463                 raw_encap_data =
3464                         (const struct rte_flow_action_raw_encap *)action->conf;
3465                 res.size = raw_encap_data->size;
3466                 memcpy(res.buf, raw_encap_data->data, res.size);
3467         } else {
3468                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3469                         encap_data =
3470                                 ((const struct rte_flow_action_vxlan_encap *)
3471                                                 action->conf)->definition;
3472                 else
3473                         encap_data =
3474                                 ((const struct rte_flow_action_nvgre_encap *)
3475                                                 action->conf)->definition;
3476                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3477                                                &res.size, error))
3478                         return -rte_errno;
3479         }
3480         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3481                 return -rte_errno;
3482         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3483                 return rte_flow_error_set(error, EINVAL,
3484                                           RTE_FLOW_ERROR_TYPE_ACTION,
3485                                           NULL, "can't create L2 encap action");
3486         return 0;
3487 }
3488
3489 /**
3490  * Convert L2 decap action to DV specification.
3491  *
3492  * @param[in] dev
3493  *   Pointer to rte_eth_dev structure.
3494  * @param[in, out] dev_flow
3495  *   Pointer to the mlx5_flow.
3496  * @param[in] transfer
3497  *   Mark if the flow is E-Switch flow.
3498  * @param[out] error
3499  *   Pointer to the error structure.
3500  *
3501  * @return
3502  *   0 on success, a negative errno value otherwise and rte_errno is set.
3503  */
3504 static int
3505 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3506                                struct mlx5_flow *dev_flow,
3507                                uint8_t transfer,
3508                                struct rte_flow_error *error)
3509 {
3510         struct mlx5_flow_dv_encap_decap_resource res = {
3511                 .size = 0,
3512                 .reformat_type =
3513                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3514                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3515                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3516         };
3517
3518         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3519                 return rte_flow_error_set(error, EINVAL,
3520                                           RTE_FLOW_ERROR_TYPE_ACTION,
3521                                           NULL, "can't create L2 decap action");
3522         return 0;
3523 }
3524
3525 /**
3526  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3527  *
3528  * @param[in] dev
3529  *   Pointer to rte_eth_dev structure.
3530  * @param[in] action
3531  *   Pointer to action structure.
3532  * @param[in, out] dev_flow
3533  *   Pointer to the mlx5_flow.
3534  * @param[in] attr
3535  *   Pointer to the flow attributes.
3536  * @param[out] error
3537  *   Pointer to the error structure.
3538  *
3539  * @return
3540  *   0 on success, a negative errno value otherwise and rte_errno is set.
3541  */
3542 static int
3543 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3544                                 const struct rte_flow_action *action,
3545                                 struct mlx5_flow *dev_flow,
3546                                 const struct rte_flow_attr *attr,
3547                                 struct rte_flow_error *error)
3548 {
3549         const struct rte_flow_action_raw_encap *encap_data;
3550         struct mlx5_flow_dv_encap_decap_resource res;
3551
3552         memset(&res, 0, sizeof(res));
3553         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3554         res.size = encap_data->size;
3555         memcpy(res.buf, encap_data->data, res.size);
3556         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3557                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3558                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3559         if (attr->transfer)
3560                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3561         else
3562                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3563                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3564         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3565                 return rte_flow_error_set(error, EINVAL,
3566                                           RTE_FLOW_ERROR_TYPE_ACTION,
3567                                           NULL, "can't create encap action");
3568         return 0;
3569 }
3570
3571 /**
3572  * Create action push VLAN.
3573  *
3574  * @param[in] dev
3575  *   Pointer to rte_eth_dev structure.
3576  * @param[in] attr
3577  *   Pointer to the flow attributes.
3578  * @param[in] vlan
3579  *   Pointer to the vlan to push to the Ethernet header.
3580  * @param[in, out] dev_flow
3581  *   Pointer to the mlx5_flow.
3582  * @param[out] error
3583  *   Pointer to the error structure.
3584  *
3585  * @return
3586  *   0 on success, a negative errno value otherwise and rte_errno is set.
3587  */
3588 static int
3589 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3590                                 const struct rte_flow_attr *attr,
3591                                 const struct rte_vlan_hdr *vlan,
3592                                 struct mlx5_flow *dev_flow,
3593                                 struct rte_flow_error *error)
3594 {
3595         struct mlx5_flow_dv_push_vlan_action_resource res;
3596
3597         memset(&res, 0, sizeof(res));
3598         res.vlan_tag =
3599                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3600                                  vlan->vlan_tci);
3601         if (attr->transfer)
3602                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3603         else
3604                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3605                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3606         return flow_dv_push_vlan_action_resource_register
3607                                             (dev, &res, dev_flow, error);
3608 }
3609
3610 static int fdb_mirror;
3611
3612 /**
3613  * Validate the modify-header actions.
3614  *
3615  * @param[in] action_flags
3616  *   Holds the actions detected until now.
3617  * @param[in] action
3618  *   Pointer to the modify action.
3619  * @param[out] error
3620  *   Pointer to error structure.
3621  *
3622  * @return
3623  *   0 on success, a negative errno value otherwise and rte_errno is set.
3624  */
3625 static int
3626 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3627                                    const struct rte_flow_action *action,
3628                                    struct rte_flow_error *error)
3629 {
3630         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3631                 return rte_flow_error_set(error, EINVAL,
3632                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3633                                           NULL, "action configuration not set");
3634         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3635                 return rte_flow_error_set(error, EINVAL,
3636                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3637                                           "can't have encap action before"
3638                                           " modify action");
3639         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3640                 return rte_flow_error_set(error, EINVAL,
3641                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3642                                           "can't support sample action before"
3643                                           " modify action for E-Switch"
3644                                           " mirroring");
3645         return 0;
3646 }
3647
3648 /**
3649  * Validate the modify-header MAC address actions.
3650  *
3651  * @param[in] action_flags
3652  *   Holds the actions detected until now.
3653  * @param[in] action
3654  *   Pointer to the modify action.
3655  * @param[in] item_flags
3656  *   Holds the items detected.
3657  * @param[out] error
3658  *   Pointer to error structure.
3659  *
3660  * @return
3661  *   0 on success, a negative errno value otherwise and rte_errno is set.
3662  */
3663 static int
3664 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3665                                    const struct rte_flow_action *action,
3666                                    const uint64_t item_flags,
3667                                    struct rte_flow_error *error)
3668 {
3669         int ret = 0;
3670
3671         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3672         if (!ret) {
3673                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3674                         return rte_flow_error_set(error, EINVAL,
3675                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3676                                                   NULL,
3677                                                   "no L2 item in pattern");
3678         }
3679         return ret;
3680 }
3681
3682 /**
3683  * Validate the modify-header IPv4 address actions.
3684  *
3685  * @param[in] action_flags
3686  *   Holds the actions detected until now.
3687  * @param[in] action
3688  *   Pointer to the modify action.
3689  * @param[in] item_flags
3690  *   Holds the items detected.
3691  * @param[out] error
3692  *   Pointer to error structure.
3693  *
3694  * @return
3695  *   0 on success, a negative errno value otherwise and rte_errno is set.
3696  */
3697 static int
3698 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3699                                     const struct rte_flow_action *action,
3700                                     const uint64_t item_flags,
3701                                     struct rte_flow_error *error)
3702 {
3703         int ret = 0;
3704         uint64_t layer;
3705
3706         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3707         if (!ret) {
3708                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3709                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3710                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3711                 if (!(item_flags & layer))
3712                         return rte_flow_error_set(error, EINVAL,
3713                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3714                                                   NULL,
3715                                                   "no ipv4 item in pattern");
3716         }
3717         return ret;
3718 }
3719
3720 /**
3721  * Validate the modify-header IPv6 address actions.
3722  *
3723  * @param[in] action_flags
3724  *   Holds the actions detected until now.
3725  * @param[in] action
3726  *   Pointer to the modify action.
3727  * @param[in] item_flags
3728  *   Holds the items detected.
3729  * @param[out] error
3730  *   Pointer to error structure.
3731  *
3732  * @return
3733  *   0 on success, a negative errno value otherwise and rte_errno is set.
3734  */
3735 static int
3736 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3737                                     const struct rte_flow_action *action,
3738                                     const uint64_t item_flags,
3739                                     struct rte_flow_error *error)
3740 {
3741         int ret = 0;
3742         uint64_t layer;
3743
3744         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3745         if (!ret) {
3746                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3747                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3748                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3749                 if (!(item_flags & layer))
3750                         return rte_flow_error_set(error, EINVAL,
3751                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3752                                                   NULL,
3753                                                   "no ipv6 item in pattern");
3754         }
3755         return ret;
3756 }
3757
3758 /**
3759  * Validate the modify-header TP actions.
3760  *
3761  * @param[in] action_flags
3762  *   Holds the actions detected until now.
3763  * @param[in] action
3764  *   Pointer to the modify action.
3765  * @param[in] item_flags
3766  *   Holds the items detected.
3767  * @param[out] error
3768  *   Pointer to error structure.
3769  *
3770  * @return
3771  *   0 on success, a negative errno value otherwise and rte_errno is set.
3772  */
3773 static int
3774 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3775                                   const struct rte_flow_action *action,
3776                                   const uint64_t item_flags,
3777                                   struct rte_flow_error *error)
3778 {
3779         int ret = 0;
3780         uint64_t layer;
3781
3782         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3783         if (!ret) {
3784                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3785                                  MLX5_FLOW_LAYER_INNER_L4 :
3786                                  MLX5_FLOW_LAYER_OUTER_L4;
3787                 if (!(item_flags & layer))
3788                         return rte_flow_error_set(error, EINVAL,
3789                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3790                                                   NULL, "no transport layer "
3791                                                   "in pattern");
3792         }
3793         return ret;
3794 }
3795
3796 /**
3797  * Validate the modify-header actions of increment/decrement
3798  * TCP Sequence-number.
3799  *
3800  * @param[in] action_flags
3801  *   Holds the actions detected until now.
3802  * @param[in] action
3803  *   Pointer to the modify action.
3804  * @param[in] item_flags
3805  *   Holds the items detected.
3806  * @param[out] error
3807  *   Pointer to error structure.
3808  *
3809  * @return
3810  *   0 on success, a negative errno value otherwise and rte_errno is set.
3811  */
3812 static int
3813 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3814                                        const struct rte_flow_action *action,
3815                                        const uint64_t item_flags,
3816                                        struct rte_flow_error *error)
3817 {
3818         int ret = 0;
3819         uint64_t layer;
3820
3821         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3822         if (!ret) {
3823                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3824                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3825                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3826                 if (!(item_flags & layer))
3827                         return rte_flow_error_set(error, EINVAL,
3828                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3829                                                   NULL, "no TCP item in"
3830                                                   " pattern");
3831                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3832                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3833                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3834                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3835                         return rte_flow_error_set(error, EINVAL,
3836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3837                                                   NULL,
3838                                                   "cannot decrease and increase"
3839                                                   " TCP sequence number"
3840                                                   " at the same time");
3841         }
3842         return ret;
3843 }
3844
3845 /**
3846  * Validate the modify-header actions of increment/decrement
3847  * TCP Acknowledgment number.
3848  *
3849  * @param[in] action_flags
3850  *   Holds the actions detected until now.
3851  * @param[in] action
3852  *   Pointer to the modify action.
3853  * @param[in] item_flags
3854  *   Holds the items detected.
3855  * @param[out] error
3856  *   Pointer to error structure.
3857  *
3858  * @return
3859  *   0 on success, a negative errno value otherwise and rte_errno is set.
3860  */
3861 static int
3862 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3863                                        const struct rte_flow_action *action,
3864                                        const uint64_t item_flags,
3865                                        struct rte_flow_error *error)
3866 {
3867         int ret = 0;
3868         uint64_t layer;
3869
3870         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3871         if (!ret) {
3872                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3873                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3874                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3875                 if (!(item_flags & layer))
3876                         return rte_flow_error_set(error, EINVAL,
3877                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3878                                                   NULL, "no TCP item in"
3879                                                   " pattern");
3880                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3881                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3882                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3883                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3884                         return rte_flow_error_set(error, EINVAL,
3885                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3886                                                   NULL,
3887                                                   "cannot decrease and increase"
3888                                                   " TCP acknowledgment number"
3889                                                   " at the same time");
3890         }
3891         return ret;
3892 }
3893
3894 /**
3895  * Validate the modify-header TTL actions.
3896  *
3897  * @param[in] action_flags
3898  *   Holds the actions detected until now.
3899  * @param[in] action
3900  *   Pointer to the modify action.
3901  * @param[in] item_flags
3902  *   Holds the items detected.
3903  * @param[out] error
3904  *   Pointer to error structure.
3905  *
3906  * @return
3907  *   0 on success, a negative errno value otherwise and rte_errno is set.
3908  */
3909 static int
3910 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3911                                    const struct rte_flow_action *action,
3912                                    const uint64_t item_flags,
3913                                    struct rte_flow_error *error)
3914 {
3915         int ret = 0;
3916         uint64_t layer;
3917
3918         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3919         if (!ret) {
3920                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3921                                  MLX5_FLOW_LAYER_INNER_L3 :
3922                                  MLX5_FLOW_LAYER_OUTER_L3;
3923                 if (!(item_flags & layer))
3924                         return rte_flow_error_set(error, EINVAL,
3925                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3926                                                   NULL,
3927                                                   "no IP protocol in pattern");
3928         }
3929         return ret;
3930 }
3931
3932 /**
3933  * Validate jump action.
3934  *
3935  * @param[in] action
3936  *   Pointer to the jump action.
3937  * @param[in] action_flags
3938  *   Holds the actions detected until now.
3939  * @param[in] attributes
3940  *   Pointer to flow attributes
3941  * @param[in] external
3942  *   Action belongs to flow rule created by request external to PMD.
3943  * @param[out] error
3944  *   Pointer to error structure.
3945  *
3946  * @return
3947  *   0 on success, a negative errno value otherwise and rte_errno is set.
3948  */
3949 static int
3950 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3951                              uint64_t action_flags,
3952                              const struct rte_flow_attr *attributes,
3953                              bool external, struct rte_flow_error *error)
3954 {
3955         uint32_t target_group, table;
3956         int ret = 0;
3957
3958         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3959                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3960                 return rte_flow_error_set(error, EINVAL,
3961                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3962                                           "can't have 2 fate actions in"
3963                                           " same flow");
3964         if (action_flags & MLX5_FLOW_ACTION_METER)
3965                 return rte_flow_error_set(error, ENOTSUP,
3966                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3967                                           "jump with meter not support");
3968         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3969                 return rte_flow_error_set(error, EINVAL,
3970                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3971                                           "E-Switch mirroring can't support"
3972                                           " Sample action and jump action in"
3973                                           " same flow now");
3974         if (!action->conf)
3975                 return rte_flow_error_set(error, EINVAL,
3976                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3977                                           NULL, "action configuration not set");
3978         target_group =
3979                 ((const struct rte_flow_action_jump *)action->conf)->group;
3980         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3981                                        true, &table, error);
3982         if (ret)
3983                 return ret;
3984         if (attributes->group == target_group)
3985                 return rte_flow_error_set(error, EINVAL,
3986                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3987                                           "target group must be other than"
3988                                           " the current flow group");
3989         return 0;
3990 }
3991
3992 /*
3993  * Validate the port_id action.
3994  *
3995  * @param[in] dev
3996  *   Pointer to rte_eth_dev structure.
3997  * @param[in] action_flags
3998  *   Bit-fields that holds the actions detected until now.
3999  * @param[in] action
4000  *   Port_id RTE action structure.
4001  * @param[in] attr
4002  *   Attributes of flow that includes this action.
4003  * @param[out] error
4004  *   Pointer to error structure.
4005  *
4006  * @return
4007  *   0 on success, a negative errno value otherwise and rte_errno is set.
4008  */
4009 static int
4010 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
4011                                 uint64_t action_flags,
4012                                 const struct rte_flow_action *action,
4013                                 const struct rte_flow_attr *attr,
4014                                 struct rte_flow_error *error)
4015 {
4016         const struct rte_flow_action_port_id *port_id;
4017         struct mlx5_priv *act_priv;
4018         struct mlx5_priv *dev_priv;
4019         uint16_t port;
4020
4021         if (!attr->transfer)
4022                 return rte_flow_error_set(error, ENOTSUP,
4023                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4024                                           NULL,
4025                                           "port id action is valid in transfer"
4026                                           " mode only");
4027         if (!action || !action->conf)
4028                 return rte_flow_error_set(error, ENOTSUP,
4029                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4030                                           NULL,
4031                                           "port id action parameters must be"
4032                                           " specified");
4033         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4034                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4035                 return rte_flow_error_set(error, EINVAL,
4036                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4037                                           "can have only one fate actions in"
4038                                           " a flow");
4039         dev_priv = mlx5_dev_to_eswitch_info(dev);
4040         if (!dev_priv)
4041                 return rte_flow_error_set(error, rte_errno,
4042                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4043                                           NULL,
4044                                           "failed to obtain E-Switch info");
4045         port_id = action->conf;
4046         port = port_id->original ? dev->data->port_id : port_id->id;
4047         act_priv = mlx5_port_to_eswitch_info(port, false);
4048         if (!act_priv)
4049                 return rte_flow_error_set
4050                                 (error, rte_errno,
4051                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
4052                                  "failed to obtain E-Switch port id for port");
4053         if (act_priv->domain_id != dev_priv->domain_id)
4054                 return rte_flow_error_set
4055                                 (error, EINVAL,
4056                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4057                                  "port does not belong to"
4058                                  " E-Switch being configured");
4059         return 0;
4060 }
4061
4062 /**
4063  * Get the maximum number of modify header actions.
4064  *
4065  * @param dev
4066  *   Pointer to rte_eth_dev structure.
4067  * @param flags
4068  *   Flags bits to check if root level.
4069  *
4070  * @return
4071  *   Max number of modify header actions device can support.
4072  */
4073 static inline unsigned int
4074 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
4075                               uint64_t flags)
4076 {
4077         /*
4078          * There's no way to directly query the max capacity from FW.
4079          * The maximal value on root table should be assumed to be supported.
4080          */
4081         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
4082                 return MLX5_MAX_MODIFY_NUM;
4083         else
4084                 return MLX5_ROOT_TBL_MODIFY_NUM;
4085 }
4086
4087 /**
4088  * Validate the meter action.
4089  *
4090  * @param[in] dev
4091  *   Pointer to rte_eth_dev structure.
4092  * @param[in] action_flags
4093  *   Bit-fields that holds the actions detected until now.
4094  * @param[in] action
4095  *   Pointer to the meter action.
4096  * @param[in] attr
4097  *   Attributes of flow that includes this action.
4098  * @param[out] error
4099  *   Pointer to error structure.
4100  *
4101  * @return
4102  *   0 on success, a negative errno value otherwise and rte_ernno is set.
4103  */
4104 static int
4105 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
4106                                 uint64_t action_flags,
4107                                 const struct rte_flow_action *action,
4108                                 const struct rte_flow_attr *attr,
4109                                 struct rte_flow_error *error)
4110 {
4111         struct mlx5_priv *priv = dev->data->dev_private;
4112         const struct rte_flow_action_meter *am = action->conf;
4113         struct mlx5_flow_meter *fm;
4114
4115         if (!am)
4116                 return rte_flow_error_set(error, EINVAL,
4117                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4118                                           "meter action conf is NULL");
4119
4120         if (action_flags & MLX5_FLOW_ACTION_METER)
4121                 return rte_flow_error_set(error, ENOTSUP,
4122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4123                                           "meter chaining not support");
4124         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4125                 return rte_flow_error_set(error, ENOTSUP,
4126                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4127                                           "meter with jump not support");
4128         if (!priv->mtr_en)
4129                 return rte_flow_error_set(error, ENOTSUP,
4130                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4131                                           NULL,
4132                                           "meter action not supported");
4133         fm = mlx5_flow_meter_find(priv, am->mtr_id);
4134         if (!fm)
4135                 return rte_flow_error_set(error, EINVAL,
4136                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4137                                           "Meter not found");
4138         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
4139               (!fm->ingress && !attr->ingress && attr->egress) ||
4140               (!fm->egress && !attr->egress && attr->ingress))))
4141                 return rte_flow_error_set(error, EINVAL,
4142                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4143                                           "Flow attributes are either invalid "
4144                                           "or have a conflict with current "
4145                                           "meter attributes");
4146         return 0;
4147 }
4148
4149 /**
4150  * Validate the age action.
4151  *
4152  * @param[in] action_flags
4153  *   Holds the actions detected until now.
4154  * @param[in] action
4155  *   Pointer to the age action.
4156  * @param[in] dev
4157  *   Pointer to the Ethernet device structure.
4158  * @param[out] error
4159  *   Pointer to error structure.
4160  *
4161  * @return
4162  *   0 on success, a negative errno value otherwise and rte_errno is set.
4163  */
4164 static int
4165 flow_dv_validate_action_age(uint64_t action_flags,
4166                             const struct rte_flow_action *action,
4167                             struct rte_eth_dev *dev,
4168                             struct rte_flow_error *error)
4169 {
4170         struct mlx5_priv *priv = dev->data->dev_private;
4171         const struct rte_flow_action_age *age = action->conf;
4172
4173         if (!priv->config.devx || priv->counter_fallback)
4174                 return rte_flow_error_set(error, ENOTSUP,
4175                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4176                                           NULL,
4177                                           "age action not supported");
4178         if (!(action->conf))
4179                 return rte_flow_error_set(error, EINVAL,
4180                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4181                                           "configuration cannot be null");
4182         if (!(age->timeout))
4183                 return rte_flow_error_set(error, EINVAL,
4184                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4185                                           "invalid timeout value 0");
4186         if (action_flags & MLX5_FLOW_ACTION_AGE)
4187                 return rte_flow_error_set(error, EINVAL,
4188                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4189                                           "duplicate age actions set");
4190         return 0;
4191 }
4192
4193 /**
4194  * Validate the modify-header IPv4 DSCP actions.
4195  *
4196  * @param[in] action_flags
4197  *   Holds the actions detected until now.
4198  * @param[in] action
4199  *   Pointer to the modify action.
4200  * @param[in] item_flags
4201  *   Holds the items detected.
4202  * @param[out] error
4203  *   Pointer to error structure.
4204  *
4205  * @return
4206  *   0 on success, a negative errno value otherwise and rte_errno is set.
4207  */
4208 static int
4209 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
4210                                          const struct rte_flow_action *action,
4211                                          const uint64_t item_flags,
4212                                          struct rte_flow_error *error)
4213 {
4214         int ret = 0;
4215
4216         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4217         if (!ret) {
4218                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
4219                         return rte_flow_error_set(error, EINVAL,
4220                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4221                                                   NULL,
4222                                                   "no ipv4 item in pattern");
4223         }
4224         return ret;
4225 }
4226
4227 /**
4228  * Validate the modify-header IPv6 DSCP actions.
4229  *
4230  * @param[in] action_flags
4231  *   Holds the actions detected until now.
4232  * @param[in] action
4233  *   Pointer to the modify action.
4234  * @param[in] item_flags
4235  *   Holds the items detected.
4236  * @param[out] error
4237  *   Pointer to error structure.
4238  *
4239  * @return
4240  *   0 on success, a negative errno value otherwise and rte_errno is set.
4241  */
4242 static int
4243 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
4244                                          const struct rte_flow_action *action,
4245                                          const uint64_t item_flags,
4246                                          struct rte_flow_error *error)
4247 {
4248         int ret = 0;
4249
4250         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4251         if (!ret) {
4252                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
4253                         return rte_flow_error_set(error, EINVAL,
4254                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4255                                                   NULL,
4256                                                   "no ipv6 item in pattern");
4257         }
4258         return ret;
4259 }
4260
4261 /**
4262  * Match modify-header resource.
4263  *
4264  * @param entry
4265  *   Pointer to exist resource entry object.
4266  * @param ctx
4267  *   Pointer to new modify-header resource.
4268  *
4269  * @return
4270  *   0 on matching, -1 otherwise.
4271  */
4272 static int
4273 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
4274 {
4275         struct mlx5_flow_dv_modify_hdr_resource *resource;
4276         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4277         uint32_t actions_len;
4278
4279         resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
4280         cache_resource = container_of(entry,
4281                                       struct mlx5_flow_dv_modify_hdr_resource,
4282                                       entry);
4283         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4284         if (resource->entry.key == cache_resource->entry.key &&
4285             resource->ft_type == cache_resource->ft_type &&
4286             resource->actions_num == cache_resource->actions_num &&
4287             resource->flags == cache_resource->flags &&
4288             !memcmp((const void *)resource->actions,
4289                     (const void *)cache_resource->actions,
4290                     actions_len))
4291                 return 0;
4292         return -1;
4293 }
4294
4295 /**
4296  * Validate the sample action.
4297  *
4298  * @param[in] action_flags
4299  *   Holds the actions detected until now.
4300  * @param[in] action
4301  *   Pointer to the sample action.
4302  * @param[in] dev
4303  *   Pointer to the Ethernet device structure.
4304  * @param[in] attr
4305  *   Attributes of flow that includes this action.
4306  * @param[out] error
4307  *   Pointer to error structure.
4308  *
4309  * @return
4310  *   0 on success, a negative errno value otherwise and rte_errno is set.
4311  */
4312 static int
4313 flow_dv_validate_action_sample(uint64_t action_flags,
4314                                const struct rte_flow_action *action,
4315                                struct rte_eth_dev *dev,
4316                                const struct rte_flow_attr *attr,
4317                                struct rte_flow_error *error)
4318 {
4319         struct mlx5_priv *priv = dev->data->dev_private;
4320         struct mlx5_dev_config *dev_conf = &priv->config;
4321         const struct rte_flow_action_sample *sample = action->conf;
4322         const struct rte_flow_action *act;
4323         uint64_t sub_action_flags = 0;
4324         uint16_t queue_index = 0xFFFF;
4325         int actions_n = 0;
4326         int ret;
4327         fdb_mirror = 0;
4328
4329         if (!sample)
4330                 return rte_flow_error_set(error, EINVAL,
4331                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4332                                           "configuration cannot be NULL");
4333         if (sample->ratio == 0)
4334                 return rte_flow_error_set(error, EINVAL,
4335                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4336                                           "ratio value starts from 1");
4337         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
4338                 return rte_flow_error_set(error, ENOTSUP,
4339                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4340                                           NULL,
4341                                           "sample action not supported");
4342         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
4343                 return rte_flow_error_set(error, EINVAL,
4344                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4345                                           "Multiple sample actions not "
4346                                           "supported");
4347         if (action_flags & MLX5_FLOW_ACTION_METER)
4348                 return rte_flow_error_set(error, EINVAL,
4349                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4350                                           "wrong action order, meter should "
4351                                           "be after sample action");
4352         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4353                 return rte_flow_error_set(error, EINVAL,
4354                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4355                                           "wrong action order, jump should "
4356                                           "be after sample action");
4357         act = sample->actions;
4358         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
4359                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4360                         return rte_flow_error_set(error, ENOTSUP,
4361                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4362                                                   act, "too many actions");
4363                 switch (act->type) {
4364                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4365                         ret = mlx5_flow_validate_action_queue(act,
4366                                                               sub_action_flags,
4367                                                               dev,
4368                                                               attr, error);
4369                         if (ret < 0)
4370                                 return ret;
4371                         queue_index = ((const struct rte_flow_action_queue *)
4372                                                         (act->conf))->index;
4373                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
4374                         ++actions_n;
4375                         break;
4376                 case RTE_FLOW_ACTION_TYPE_MARK:
4377                         ret = flow_dv_validate_action_mark(dev, act,
4378                                                            sub_action_flags,
4379                                                            attr, error);
4380                         if (ret < 0)
4381                                 return ret;
4382                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4383                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
4384                                                 MLX5_FLOW_ACTION_MARK_EXT;
4385                         else
4386                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
4387                         ++actions_n;
4388                         break;
4389                 case RTE_FLOW_ACTION_TYPE_COUNT:
4390                         ret = flow_dv_validate_action_count(dev, error);
4391                         if (ret < 0)
4392                                 return ret;
4393                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
4394                         ++actions_n;
4395                         break;
4396                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4397                         ret = flow_dv_validate_action_port_id(dev,
4398                                                               sub_action_flags,
4399                                                               act,
4400                                                               attr,
4401                                                               error);
4402                         if (ret)
4403                                 return ret;
4404                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4405                         ++actions_n;
4406                         break;
4407                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4408                         ret = flow_dv_validate_action_raw_encap_decap
4409                                 (dev, NULL, act->conf, attr, &sub_action_flags,
4410                                  &actions_n, error);
4411                         if (ret < 0)
4412                                 return ret;
4413                         ++actions_n;
4414                         break;
4415                 default:
4416                         return rte_flow_error_set(error, ENOTSUP,
4417                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4418                                                   NULL,
4419                                                   "Doesn't support optional "
4420                                                   "action");
4421                 }
4422         }
4423         if (attr->ingress && !attr->transfer) {
4424                 if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE))
4425                         return rte_flow_error_set(error, EINVAL,
4426                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4427                                                   NULL,
4428                                                   "Ingress must has a dest "
4429                                                   "QUEUE for Sample");
4430         } else if (attr->egress && !attr->transfer) {
4431                 return rte_flow_error_set(error, ENOTSUP,
4432                                           RTE_FLOW_ERROR_TYPE_ACTION,
4433                                           NULL,
4434                                           "Sample Only support Ingress "
4435                                           "or E-Switch");
4436         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
4437                 MLX5_ASSERT(attr->transfer);
4438                 if (sample->ratio > 1)
4439                         return rte_flow_error_set(error, ENOTSUP,
4440                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4441                                                   NULL,
4442                                                   "E-Switch doesn't support "
4443                                                   "any optional action "
4444                                                   "for sampling");
4445                 fdb_mirror = 1;
4446                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
4447                         return rte_flow_error_set(error, ENOTSUP,
4448                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4449                                                   NULL,
4450                                                   "unsupported action QUEUE");
4451                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
4452                         return rte_flow_error_set(error, EINVAL,
4453                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4454                                                   NULL,
4455                                                   "E-Switch must has a dest "
4456                                                   "port for mirroring");
4457         }
4458         /* Continue validation for Xcap actions.*/
4459         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
4460             (queue_index == 0xFFFF ||
4461              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
4462                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
4463                      MLX5_FLOW_XCAP_ACTIONS)
4464                         return rte_flow_error_set(error, ENOTSUP,
4465                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4466                                                   NULL, "encap and decap "
4467                                                   "combination aren't "
4468                                                   "supported");
4469                 if (!attr->transfer && attr->ingress && (sub_action_flags &
4470                                                         MLX5_FLOW_ACTION_ENCAP))
4471                         return rte_flow_error_set(error, ENOTSUP,
4472                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4473                                                   NULL, "encap is not supported"
4474                                                   " for ingress traffic");
4475         }
4476         return 0;
4477 }
4478
4479 /**
4480  * Find existing modify-header resource or create and register a new one.
4481  *
4482  * @param dev[in, out]
4483  *   Pointer to rte_eth_dev structure.
4484  * @param[in, out] resource
4485  *   Pointer to modify-header resource.
4486  * @parm[in, out] dev_flow
4487  *   Pointer to the dev_flow.
4488  * @param[out] error
4489  *   pointer to error structure.
4490  *
4491  * @return
4492  *   0 on success otherwise -errno and errno is set.
4493  */
4494 static int
4495 flow_dv_modify_hdr_resource_register
4496                         (struct rte_eth_dev *dev,
4497                          struct mlx5_flow_dv_modify_hdr_resource *resource,
4498                          struct mlx5_flow *dev_flow,
4499                          struct rte_flow_error *error)
4500 {
4501         struct mlx5_priv *priv = dev->data->dev_private;
4502         struct mlx5_dev_ctx_shared *sh = priv->sh;
4503         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4504         struct mlx5dv_dr_domain *ns;
4505         uint32_t actions_len;
4506         struct mlx5_hlist_entry *entry;
4507         union mlx5_flow_modify_hdr_key hdr_mod_key = {
4508                 {
4509                         .ft_type = resource->ft_type,
4510                         .actions_num = resource->actions_num,
4511                         .group = dev_flow->dv.group,
4512                         .cksum = 0,
4513                 }
4514         };
4515         int ret;
4516
4517         resource->flags = dev_flow->dv.group ? 0 :
4518                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4519         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4520                                     resource->flags))
4521                 return rte_flow_error_set(error, EOVERFLOW,
4522                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4523                                           "too many modify header items");
4524         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4525                 ns = sh->fdb_domain;
4526         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4527                 ns = sh->tx_domain;
4528         else
4529                 ns = sh->rx_domain;
4530         /* Lookup a matching resource from cache. */
4531         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4532         hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
4533         resource->entry.key = hdr_mod_key.v64;
4534         entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
4535                                      flow_dv_modify_hdr_resource_match,
4536                                      (void *)resource);
4537         if (entry) {
4538                 cache_resource = container_of(entry,
4539                                         struct mlx5_flow_dv_modify_hdr_resource,
4540                                         entry);
4541                 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4542                         (void *)cache_resource,
4543                         rte_atomic32_read(&cache_resource->refcnt));
4544                 rte_atomic32_inc(&cache_resource->refcnt);
4545                 dev_flow->handle->dvh.modify_hdr = cache_resource;
4546                 return 0;
4547
4548         }
4549         /* Register new modify-header resource. */
4550         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4551                                     sizeof(*cache_resource) + actions_len, 0,
4552                                     SOCKET_ID_ANY);
4553         if (!cache_resource)
4554                 return rte_flow_error_set(error, ENOMEM,
4555                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4556                                           "cannot allocate resource memory");
4557         *cache_resource = *resource;
4558         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4559         ret = mlx5_flow_os_create_flow_action_modify_header
4560                                         (sh->ctx, ns, cache_resource,
4561                                          actions_len, &cache_resource->action);
4562         if (ret) {
4563                 mlx5_free(cache_resource);
4564                 return rte_flow_error_set(error, ENOMEM,
4565                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4566                                           NULL, "cannot create action");
4567         }
4568         rte_atomic32_init(&cache_resource->refcnt);
4569         rte_atomic32_inc(&cache_resource->refcnt);
4570         if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
4571                                  flow_dv_modify_hdr_resource_match,
4572                                  (void *)cache_resource)) {
4573                 claim_zero(mlx5_flow_os_destroy_flow_action
4574                                                 (cache_resource->action));
4575                 mlx5_free(cache_resource);
4576                 return rte_flow_error_set(error, EEXIST,
4577                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4578                                           NULL, "action exist");
4579         }
4580         dev_flow->handle->dvh.modify_hdr = cache_resource;
4581         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4582                 (void *)cache_resource,
4583                 rte_atomic32_read(&cache_resource->refcnt));
4584         return 0;
4585 }
4586
4587 /**
4588  * Get DV flow counter by index.
4589  *
4590  * @param[in] dev
4591  *   Pointer to the Ethernet device structure.
4592  * @param[in] idx
4593  *   mlx5 flow counter index in the container.
4594  * @param[out] ppool
4595  *   mlx5 flow counter pool in the container,
4596  *
4597  * @return
4598  *   Pointer to the counter, NULL otherwise.
4599  */
4600 static struct mlx5_flow_counter *
4601 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4602                            uint32_t idx,
4603                            struct mlx5_flow_counter_pool **ppool)
4604 {
4605         struct mlx5_priv *priv = dev->data->dev_private;
4606         struct mlx5_pools_container *cont;
4607         struct mlx5_flow_counter_pool *pool;
4608         uint32_t batch = 0, age = 0;
4609
4610         idx--;
4611         age = MLX_CNT_IS_AGE(idx);
4612         idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4613         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4614                 idx -= MLX5_CNT_BATCH_OFFSET;
4615                 batch = 1;
4616         }
4617         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4618         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4619         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4620         MLX5_ASSERT(pool);
4621         if (ppool)
4622                 *ppool = pool;
4623         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4624 }
4625
4626 /**
4627  * Check the devx counter belongs to the pool.
4628  *
4629  * @param[in] pool
4630  *   Pointer to the counter pool.
4631  * @param[in] id
4632  *   The counter devx ID.
4633  *
4634  * @return
4635  *   True if counter belongs to the pool, false otherwise.
4636  */
4637 static bool
4638 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4639 {
4640         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4641                    MLX5_COUNTERS_PER_POOL;
4642
4643         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4644                 return true;
4645         return false;
4646 }
4647
4648 /**
4649  * Get a pool by devx counter ID.
4650  *
4651  * @param[in] cont
4652  *   Pointer to the counter container.
4653  * @param[in] id
4654  *   The counter devx ID.
4655  *
4656  * @return
4657  *   The counter pool pointer if exists, NULL otherwise,
4658  */
4659 static struct mlx5_flow_counter_pool *
4660 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4661 {
4662         uint32_t i;
4663
4664         /* Check last used pool. */
4665         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4666             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4667                 return cont->pools[cont->last_pool_idx];
4668         /* ID out of range means no suitable pool in the container. */
4669         if (id > cont->max_id || id < cont->min_id)
4670                 return NULL;
4671         /*
4672          * Find the pool from the end of the container, since mostly counter
4673          * ID is sequence increasing, and the last pool should be the needed
4674          * one.
4675          */
4676         i = rte_atomic16_read(&cont->n_valid);
4677         while (i--) {
4678                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4679
4680                 if (flow_dv_is_counter_in_pool(pool, id))
4681                         return pool;
4682         }
4683         return NULL;
4684 }
4685
4686 /**
4687  * Allocate a new memory for the counter values wrapped by all the needed
4688  * management.
4689  *
4690  * @param[in] dev
4691  *   Pointer to the Ethernet device structure.
4692  * @param[in] raws_n
4693  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4694  *
4695  * @return
4696  *   The new memory management pointer on success, otherwise NULL and rte_errno
4697  *   is set.
4698  */
4699 static struct mlx5_counter_stats_mem_mng *
4700 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4701 {
4702         struct mlx5_priv *priv = dev->data->dev_private;
4703         struct mlx5_dev_ctx_shared *sh = priv->sh;
4704         struct mlx5_devx_mkey_attr mkey_attr;
4705         struct mlx5_counter_stats_mem_mng *mem_mng;
4706         volatile struct flow_counter_stats *raw_data;
4707         int size = (sizeof(struct flow_counter_stats) *
4708                         MLX5_COUNTERS_PER_POOL +
4709                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4710                         sizeof(struct mlx5_counter_stats_mem_mng);
4711         size_t pgsize = rte_mem_page_size();
4712         if (pgsize == (size_t)-1) {
4713                 DRV_LOG(ERR, "Failed to get mem page size");
4714                 rte_errno = ENOMEM;
4715                 return NULL;
4716         }
4717         uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
4718                                   SOCKET_ID_ANY);
4719         int i;
4720
4721         if (!mem) {
4722                 rte_errno = ENOMEM;
4723                 return NULL;
4724         }
4725         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4726         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4727         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4728                                                  IBV_ACCESS_LOCAL_WRITE);
4729         if (!mem_mng->umem) {
4730                 rte_errno = errno;
4731                 mlx5_free(mem);
4732                 return NULL;
4733         }
4734         mkey_attr.addr = (uintptr_t)mem;
4735         mkey_attr.size = size;
4736         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4737         mkey_attr.pd = sh->pdn;
4738         mkey_attr.log_entity_size = 0;
4739         mkey_attr.pg_access = 0;
4740         mkey_attr.klm_array = NULL;
4741         mkey_attr.klm_num = 0;
4742         if (priv->config.hca_attr.relaxed_ordering_write &&
4743                 priv->config.hca_attr.relaxed_ordering_read  &&
4744                 !haswell_broadwell_cpu)
4745                 mkey_attr.relaxed_ordering = 1;
4746         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4747         if (!mem_mng->dm) {
4748                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4749                 rte_errno = errno;
4750                 mlx5_free(mem);
4751                 return NULL;
4752         }
4753         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4754         raw_data = (volatile struct flow_counter_stats *)mem;
4755         for (i = 0; i < raws_n; ++i) {
4756                 mem_mng->raws[i].mem_mng = mem_mng;
4757                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4758         }
4759         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4760         return mem_mng;
4761 }
4762
4763 /**
4764  * Resize a counter container.
4765  *
4766  * @param[in] dev
4767  *   Pointer to the Ethernet device structure.
4768  * @param[in] batch
4769  *   Whether the pool is for counter that was allocated by batch command.
4770  * @param[in] age
4771  *   Whether the pool is for Aging counter.
4772  *
4773  * @return
4774  *   0 on success, otherwise negative errno value and rte_errno is set.
4775  */
4776 static int
4777 flow_dv_container_resize(struct rte_eth_dev *dev,
4778                                 uint32_t batch, uint32_t age)
4779 {
4780         struct mlx5_priv *priv = dev->data->dev_private;
4781         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4782                                                                age);
4783         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4784         void *old_pools = cont->pools;
4785         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4786         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4787         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4788
4789         if (!pools) {
4790                 rte_errno = ENOMEM;
4791                 return -ENOMEM;
4792         }
4793         if (old_pools)
4794                 memcpy(pools, old_pools, cont->n *
4795                                        sizeof(struct mlx5_flow_counter_pool *));
4796         /*
4797          * Fallback mode query the counter directly, no background query
4798          * resources are needed.
4799          */
4800         if (!priv->counter_fallback) {
4801                 int i;
4802
4803                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4804                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4805                 if (!mem_mng) {
4806                         mlx5_free(pools);
4807                         return -ENOMEM;
4808                 }
4809                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4810                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4811                                          mem_mng->raws +
4812                                          MLX5_CNT_CONTAINER_RESIZE +
4813                                          i, next);
4814         }
4815         rte_spinlock_lock(&cont->resize_sl);
4816         cont->n = resize;
4817         cont->mem_mng = mem_mng;
4818         cont->pools = pools;
4819         rte_spinlock_unlock(&cont->resize_sl);
4820         if (old_pools)
4821                 mlx5_free(old_pools);
4822         return 0;
4823 }
4824
4825 /**
4826  * Query a devx flow counter.
4827  *
4828  * @param[in] dev
4829  *   Pointer to the Ethernet device structure.
4830  * @param[in] cnt
4831  *   Index to the flow counter.
4832  * @param[out] pkts
4833  *   The statistics value of packets.
4834  * @param[out] bytes
4835  *   The statistics value of bytes.
4836  *
4837  * @return
4838  *   0 on success, otherwise a negative errno value and rte_errno is set.
4839  */
4840 static inline int
4841 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4842                      uint64_t *bytes)
4843 {
4844         struct mlx5_priv *priv = dev->data->dev_private;
4845         struct mlx5_flow_counter_pool *pool = NULL;
4846         struct mlx5_flow_counter *cnt;
4847         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4848         int offset;
4849
4850         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4851         MLX5_ASSERT(pool);
4852         if (counter < MLX5_CNT_BATCH_OFFSET) {
4853                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4854                 if (priv->counter_fallback)
4855                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4856                                         0, pkts, bytes, 0, NULL, NULL, 0);
4857         }
4858
4859         rte_spinlock_lock(&pool->sl);
4860         /*
4861          * The single counters allocation may allocate smaller ID than the
4862          * current allocated in parallel to the host reading.
4863          * In this case the new counter values must be reported as 0.
4864          */
4865         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4866                 *pkts = 0;
4867                 *bytes = 0;
4868         } else {
4869                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4870                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4871                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4872         }
4873         rte_spinlock_unlock(&pool->sl);
4874         return 0;
4875 }
4876
4877 /**
4878  * Create and initialize a new counter pool.
4879  *
4880  * @param[in] dev
4881  *   Pointer to the Ethernet device structure.
4882  * @param[out] dcs
4883  *   The devX counter handle.
4884  * @param[in] batch
4885  *   Whether the pool is for counter that was allocated by batch command.
4886  * @param[in] age
4887  *   Whether the pool is for counter that was allocated for aging.
4888  * @param[in/out] cont_cur
4889  *   Pointer to the container pointer, it will be update in pool resize.
4890  *
4891  * @return
4892  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4893  */
4894 static struct mlx5_flow_counter_pool *
4895 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4896                     uint32_t batch, uint32_t age)
4897 {
4898         struct mlx5_priv *priv = dev->data->dev_private;
4899         struct mlx5_flow_counter_pool *pool;
4900         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4901                                                                age);
4902         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4903         uint32_t size = sizeof(*pool);
4904
4905         if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4906                 return NULL;
4907         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4908         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4909         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4910         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4911         if (!pool) {
4912                 rte_errno = ENOMEM;
4913                 return NULL;
4914         }
4915         pool->min_dcs = dcs;
4916         if (!priv->counter_fallback)
4917                 pool->raw = cont->mem_mng->raws + n_valid %
4918                                                       MLX5_CNT_CONTAINER_RESIZE;
4919         pool->raw_hw = NULL;
4920         pool->type = 0;
4921         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4922         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4923         pool->query_gen = 0;
4924         rte_spinlock_init(&pool->sl);
4925         TAILQ_INIT(&pool->counters[0]);
4926         TAILQ_INIT(&pool->counters[1]);
4927         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4928         pool->index = n_valid;
4929         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
4930         cont->pools[n_valid] = pool;
4931         if (!batch) {
4932                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4933
4934                 if (base < cont->min_id)
4935                         cont->min_id = base;
4936                 if (base > cont->max_id)
4937                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4938                 cont->last_pool_idx = pool->index;
4939         }
4940         /* Pool initialization must be updated before host thread access. */
4941         rte_io_wmb();
4942         rte_atomic16_add(&cont->n_valid, 1);
4943         return pool;
4944 }
4945
4946 /**
4947  * Restore skipped counters in the pool.
4948  *
4949  * As counter pool query requires the first counter dcs
4950  * ID start with 4 alinged, if the pool counters with
4951  * min_dcs ID are not aligned with 4, the counters will
4952  * be skipped.
4953  * Once other min_dcs ID less than these skipped counter
4954  * dcs ID appears, the skipped counters will be safe to
4955  * use.
4956  * Should be called when min_dcs is updated.
4957  *
4958  * @param[in] pool
4959  *   Current counter pool.
4960  * @param[in] last_min_dcs
4961  *   Last min_dcs.
4962  */
4963 static void
4964 flow_dv_counter_restore(struct mlx5_flow_counter_pool *pool,
4965                         struct mlx5_devx_obj *last_min_dcs)
4966 {
4967         struct mlx5_flow_counter_ext *cnt_ext;
4968         uint32_t offset, new_offset;
4969         uint32_t skip_cnt = 0;
4970         uint32_t i;
4971
4972         if (!pool->skip_cnt)
4973                 return;
4974         /*
4975          * If last min_dcs is not valid. The skipped counter may even after
4976          * last min_dcs, set the offset to the whole pool.
4977          */
4978         if (last_min_dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
4979                 offset = MLX5_COUNTERS_PER_POOL;
4980         else
4981                 offset = last_min_dcs->id % MLX5_COUNTERS_PER_POOL;
4982         new_offset = pool->min_dcs->id % MLX5_COUNTERS_PER_POOL;
4983         /*
4984          * Check the counters from 1 to the last_min_dcs range. Counters
4985          * before new min_dcs indicates pool still has skipped counters.
4986          * Counters be skipped after new min_dcs will be ready to use.
4987          * Offset 0 counter must be empty or min_dcs, start from 1.
4988          */
4989         for (i = 1; i < offset; i++) {
4990                 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
4991                 if (cnt_ext->skipped) {
4992                         if (i > new_offset) {
4993                                 cnt_ext->skipped = 0;
4994                                 TAILQ_INSERT_TAIL
4995                                         (&pool->counters[pool->query_gen],
4996                                          MLX5_POOL_GET_CNT(pool, i), next);
4997                         } else {
4998                                 skip_cnt++;
4999                         }
5000                 }
5001         }
5002         if (!skip_cnt)
5003                 pool->skip_cnt = 0;
5004 }
5005
5006 /**
5007  * Prepare a new counter and/or a new counter pool.
5008  *
5009  * @param[in] dev
5010  *   Pointer to the Ethernet device structure.
5011  * @param[out] cnt_free
5012  *   Where to put the pointer of a new counter.
5013  * @param[in] batch
5014  *   Whether the pool is for counter that was allocated by batch command.
5015  * @param[in] age
5016  *   Whether the pool is for counter that was allocated for aging.
5017  *
5018  * @return
5019  *   The counter pool pointer and @p cnt_free is set on success,
5020  *   NULL otherwise and rte_errno is set.
5021  */
5022 static struct mlx5_flow_counter_pool *
5023 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
5024                              struct mlx5_flow_counter **cnt_free,
5025                              uint32_t batch, uint32_t age)
5026 {
5027         struct mlx5_priv *priv = dev->data->dev_private;
5028         struct mlx5_pools_container *cont;
5029         struct mlx5_flow_counter_pool *pool;
5030         struct mlx5_counters tmp_tq;
5031         struct mlx5_devx_obj *last_min_dcs;
5032         struct mlx5_devx_obj *dcs = NULL;
5033         struct mlx5_flow_counter *cnt;
5034         uint32_t add2other;
5035         uint32_t i;
5036
5037         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
5038         if (!batch) {
5039 retry:
5040                 add2other = 0;
5041                 /* bulk_bitmap must be 0 for single counter allocation. */
5042                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
5043                 if (!dcs)
5044                         return NULL;
5045                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
5046                 /* Check if counter belongs to exist pool ID range. */
5047                 if (!pool) {
5048                         pool = flow_dv_find_pool_by_id
5049                                (MLX5_CNT_CONTAINER
5050                                (priv->sh, batch, (age ^ 0x1)), dcs->id);
5051                         /*
5052                          * Pool exists, counter will be added to the other
5053                          * container, need to reallocate it later.
5054                          */
5055                         if (pool) {
5056                                 add2other = 1;
5057                         } else {
5058                                 pool = flow_dv_pool_create(dev, dcs, batch,
5059                                                            age);
5060                                 if (!pool) {
5061                                         mlx5_devx_cmd_destroy(dcs);
5062                                         return NULL;
5063                                 }
5064                         }
5065                 }
5066                 if ((dcs->id < pool->min_dcs->id ||
5067                     pool->min_dcs->id &
5068                     (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) &&
5069                     !(dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))) {
5070                         /*
5071                          * Update the pool min_dcs only if current dcs is
5072                          * valid and exist min_dcs is not valid or greater
5073                          * than new dcs.
5074                          */
5075                         last_min_dcs = pool->min_dcs;
5076                         rte_atomic64_set(&pool->a64_dcs,
5077                                          (int64_t)(uintptr_t)dcs);
5078                         /*
5079                          * Restore any skipped counters if the new min_dcs
5080                          * ID is smaller or min_dcs is not valid.
5081                          */
5082                         if (dcs->id < last_min_dcs->id ||
5083                             last_min_dcs->id &
5084                             (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
5085                                 flow_dv_counter_restore(pool, last_min_dcs);
5086                 }
5087                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
5088                 cnt = MLX5_POOL_GET_CNT(pool, i);
5089                 cnt->pool = pool;
5090                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
5091                 /*
5092                  * If min_dcs is not valid, it means the new allocated dcs
5093                  * also fail to become the valid min_dcs, just skip it.
5094                  * Or if min_dcs is valid, and new dcs ID is smaller than
5095                  * min_dcs, but not become the min_dcs, also skip it.
5096                  */
5097                 if (pool->min_dcs->id &
5098                     (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1) ||
5099                     dcs->id < pool->min_dcs->id) {
5100                         MLX5_GET_POOL_CNT_EXT(pool, i)->skipped = 1;
5101                         pool->skip_cnt = 1;
5102                         goto retry;
5103                 }
5104                 if (add2other) {
5105                         TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen],
5106                                           cnt, next);
5107                         goto retry;
5108                 }
5109                 *cnt_free = cnt;
5110                 return pool;
5111         }
5112         /* bulk_bitmap is in 128 counters units. */
5113         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
5114                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
5115         if (!dcs) {
5116                 rte_errno = ENODATA;
5117                 return NULL;
5118         }
5119         pool = flow_dv_pool_create(dev, dcs, batch, age);
5120         if (!pool) {
5121                 mlx5_devx_cmd_destroy(dcs);
5122                 return NULL;
5123         }
5124         TAILQ_INIT(&tmp_tq);
5125         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
5126                 cnt = MLX5_POOL_GET_CNT(pool, i);
5127                 cnt->pool = pool;
5128                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
5129         }
5130         rte_spinlock_lock(&cont->csl);
5131         TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
5132         rte_spinlock_unlock(&cont->csl);
5133         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
5134         (*cnt_free)->pool = pool;
5135         return pool;
5136 }
5137
5138 /**
5139  * Search for existed shared counter.
5140  *
5141  * @param[in] dev
5142  *   Pointer to the Ethernet device structure.
5143  * @param[in] id
5144  *   The shared counter ID to search.
5145  * @param[out] ppool
5146  *   mlx5 flow counter pool in the container,
5147  *
5148  * @return
5149  *   NULL if not existed, otherwise pointer to the shared extend counter.
5150  */
5151 static struct mlx5_flow_counter_ext *
5152 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
5153                               struct mlx5_flow_counter_pool **ppool)
5154 {
5155         struct mlx5_priv *priv = dev->data->dev_private;
5156         union mlx5_l3t_data data;
5157         uint32_t cnt_idx;
5158
5159         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
5160                 return NULL;
5161         cnt_idx = data.dword;
5162         /*
5163          * Shared counters don't have age info. The counter extend is after
5164          * the counter datat structure.
5165          */
5166         return (struct mlx5_flow_counter_ext *)
5167                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
5168 }
5169
5170 /**
5171  * Allocate a flow counter.
5172  *
5173  * @param[in] dev
5174  *   Pointer to the Ethernet device structure.
5175  * @param[in] shared
5176  *   Indicate if this counter is shared with other flows.
5177  * @param[in] id
5178  *   Counter identifier.
5179  * @param[in] group
5180  *   Counter flow group.
5181  * @param[in] age
5182  *   Whether the counter was allocated for aging.
5183  *
5184  * @return
5185  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5186  */
5187 static uint32_t
5188 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
5189                       uint16_t group, uint32_t age)
5190 {
5191         struct mlx5_priv *priv = dev->data->dev_private;
5192         struct mlx5_flow_counter_pool *pool = NULL;
5193         struct mlx5_flow_counter *cnt_free = NULL;
5194         struct mlx5_flow_counter_ext *cnt_ext = NULL;
5195         /*
5196          * Currently group 0 flow counter cannot be assigned to a flow if it is
5197          * not the first one in the batch counter allocation, so it is better
5198          * to allocate counters one by one for these flows in a separate
5199          * container.
5200          * A counter can be shared between different groups so need to take
5201          * shared counters from the single container.
5202          */
5203         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
5204         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
5205                                                                age);
5206         uint32_t cnt_idx;
5207
5208         if (!priv->config.devx) {
5209                 rte_errno = ENOTSUP;
5210                 return 0;
5211         }
5212         if (shared) {
5213                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
5214                 if (cnt_ext) {
5215                         if (cnt_ext->ref_cnt + 1 == 0) {
5216                                 rte_errno = E2BIG;
5217                                 return 0;
5218                         }
5219                         cnt_ext->ref_cnt++;
5220                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
5221                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
5222                                   + 1;
5223                         return cnt_idx;
5224                 }
5225         }
5226         /* Get free counters from container. */
5227         rte_spinlock_lock(&cont->csl);
5228         cnt_free = TAILQ_FIRST(&cont->counters);
5229         if (cnt_free)
5230                 TAILQ_REMOVE(&cont->counters, cnt_free, next);
5231         rte_spinlock_unlock(&cont->csl);
5232         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
5233                                                        batch, age))
5234                 goto err;
5235         pool = cnt_free->pool;
5236         if (!batch)
5237                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
5238         /* Create a DV counter action only in the first time usage. */
5239         if (!cnt_free->action) {
5240                 uint16_t offset;
5241                 struct mlx5_devx_obj *dcs;
5242                 int ret;
5243
5244                 if (batch) {
5245                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
5246                         dcs = pool->min_dcs;
5247                 } else {
5248                         offset = 0;
5249                         dcs = cnt_ext->dcs;
5250                 }
5251                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
5252                                                             &cnt_free->action);
5253                 if (ret) {
5254                         rte_errno = errno;
5255                         goto err;
5256                 }
5257         }
5258         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
5259                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
5260         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
5261         cnt_idx += age * MLX5_CNT_AGE_OFFSET;
5262         /* Update the counter reset values. */
5263         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
5264                                  &cnt_free->bytes))
5265                 goto err;
5266         if (cnt_ext) {
5267                 cnt_ext->shared = shared;
5268                 cnt_ext->ref_cnt = 1;
5269                 cnt_ext->id = id;
5270                 if (shared) {
5271                         union mlx5_l3t_data data;
5272
5273                         data.dword = cnt_idx;
5274                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
5275                                 return 0;
5276                 }
5277         }
5278         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
5279                 /* Start the asynchronous batch query by the host thread. */
5280                 mlx5_set_query_alarm(priv->sh);
5281         return cnt_idx;
5282 err:
5283         if (cnt_free) {
5284                 cnt_free->pool = pool;
5285                 rte_spinlock_lock(&cont->csl);
5286                 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
5287                 rte_spinlock_unlock(&cont->csl);
5288         }
5289         return 0;
5290 }
5291
5292 /**
5293  * Get age param from counter index.
5294  *
5295  * @param[in] dev
5296  *   Pointer to the Ethernet device structure.
5297  * @param[in] counter
5298  *   Index to the counter handler.
5299  *
5300  * @return
5301  *   The aging parameter specified for the counter index.
5302  */
5303 static struct mlx5_age_param*
5304 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
5305                                 uint32_t counter)
5306 {
5307         struct mlx5_flow_counter *cnt;
5308         struct mlx5_flow_counter_pool *pool = NULL;
5309
5310         flow_dv_counter_get_by_idx(dev, counter, &pool);
5311         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
5312         cnt = MLX5_POOL_GET_CNT(pool, counter);
5313         return MLX5_CNT_TO_AGE(cnt);
5314 }
5315
5316 /**
5317  * Remove a flow counter from aged counter list.
5318  *
5319  * @param[in] dev
5320  *   Pointer to the Ethernet device structure.
5321  * @param[in] counter
5322  *   Index to the counter handler.
5323  * @param[in] cnt
5324  *   Pointer to the counter handler.
5325  */
5326 static void
5327 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
5328                                 uint32_t counter, struct mlx5_flow_counter *cnt)
5329 {
5330         struct mlx5_age_info *age_info;
5331         struct mlx5_age_param *age_param;
5332         struct mlx5_priv *priv = dev->data->dev_private;
5333         uint16_t expected = AGE_CANDIDATE;
5334
5335         age_info = GET_PORT_AGE_INFO(priv);
5336         age_param = flow_dv_counter_idx_get_age(dev, counter);
5337         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
5338                                          AGE_FREE, false, __ATOMIC_RELAXED,
5339                                          __ATOMIC_RELAXED)) {
5340                 /**
5341                  * We need the lock even it is age timeout,
5342                  * since counter may still in process.
5343                  */
5344                 rte_spinlock_lock(&age_info->aged_sl);
5345                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
5346                 rte_spinlock_unlock(&age_info->aged_sl);
5347                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
5348         }
5349 }
5350
5351 /**
5352  * Release a flow counter.
5353  *
5354  * @param[in] dev
5355  *   Pointer to the Ethernet device structure.
5356  * @param[in] counter
5357  *   Index to the counter handler.
5358  */
5359 static void
5360 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
5361 {
5362         struct mlx5_priv *priv = dev->data->dev_private;
5363         struct mlx5_flow_counter_pool *pool = NULL;
5364         struct mlx5_flow_counter *cnt;
5365         struct mlx5_flow_counter_ext *cnt_ext = NULL;
5366
5367         if (!counter)
5368                 return;
5369         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5370         MLX5_ASSERT(pool);
5371         if (counter < MLX5_CNT_BATCH_OFFSET) {
5372                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
5373                 if (cnt_ext) {
5374                         if (--cnt_ext->ref_cnt)
5375                                 return;
5376                         if (cnt_ext->shared)
5377                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
5378                                                      cnt_ext->id);
5379                 }
5380         }
5381         if (IS_AGE_POOL(pool))
5382                 flow_dv_counter_remove_from_age(dev, counter, cnt);
5383         cnt->pool = pool;
5384         /*
5385          * Put the counter back to list to be updated in none fallback mode.
5386          * Currently, we are using two list alternately, while one is in query,
5387          * add the freed counter to the other list based on the pool query_gen
5388          * value. After query finishes, add counter the list to the global
5389          * container counter list. The list changes while query starts. In
5390          * this case, lock will not be needed as query callback and release
5391          * function both operate with the different list.
5392          *
5393          */
5394         if (!priv->counter_fallback)
5395                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
5396         else
5397                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
5398                                   (priv->sh, 0, 0))->counters),
5399                                   cnt, next);
5400 }
5401
5402 /**
5403  * Verify the @p attributes will be correctly understood by the NIC and store
5404  * them in the @p flow if everything is correct.
5405  *
5406  * @param[in] dev
5407  *   Pointer to dev struct.
5408  * @param[in] attributes
5409  *   Pointer to flow attributes
5410  * @param[in] external
5411  *   This flow rule is created by request external to PMD.
5412  * @param[out] error
5413  *   Pointer to error structure.
5414  *
5415  * @return
5416  *   - 0 on success and non root table.
5417  *   - 1 on success and root table.
5418  *   - a negative errno value otherwise and rte_errno is set.
5419  */
5420 static int
5421 flow_dv_validate_attributes(struct rte_eth_dev *dev,
5422                             const struct rte_flow_attr *attributes,
5423                             bool external __rte_unused,
5424                             struct rte_flow_error *error)
5425 {
5426         struct mlx5_priv *priv = dev->data->dev_private;
5427         uint32_t priority_max = priv->config.flow_prio - 1;
5428         int ret = 0;
5429
5430 #ifndef HAVE_MLX5DV_DR
5431         if (attributes->group)
5432                 return rte_flow_error_set(error, ENOTSUP,
5433                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5434                                           NULL,
5435                                           "groups are not supported");
5436 #else
5437         uint32_t table = 0;
5438
5439         ret = mlx5_flow_group_to_table(attributes, external,
5440                                        attributes->group, !!priv->fdb_def_rule,
5441                                        &table, error);
5442         if (ret)
5443                 return ret;
5444         if (!table)
5445                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5446 #endif
5447         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
5448             attributes->priority >= priority_max)
5449                 return rte_flow_error_set(error, ENOTSUP,
5450                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
5451                                           NULL,
5452                                           "priority out of range");
5453         if (attributes->transfer) {
5454                 if (!priv->config.dv_esw_en)
5455                         return rte_flow_error_set
5456                                 (error, ENOTSUP,
5457                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5458                                  "E-Switch dr is not supported");
5459                 if (!(priv->representor || priv->master))
5460                         return rte_flow_error_set
5461                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5462                                  NULL, "E-Switch configuration can only be"
5463                                  " done by a master or a representor device");
5464                 if (attributes->egress)
5465                         return rte_flow_error_set
5466                                 (error, ENOTSUP,
5467                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
5468                                  "egress is not supported");
5469         }
5470         if (!(attributes->egress ^ attributes->ingress))
5471                 return rte_flow_error_set(error, ENOTSUP,
5472                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
5473                                           "must specify exactly one of "
5474                                           "ingress or egress");
5475         return ret;
5476 }
5477
5478 /**
5479  * Internal validation function. For validating both actions and items.
5480  *
5481  * @param[in] dev
5482  *   Pointer to the rte_eth_dev structure.
5483  * @param[in] attr
5484  *   Pointer to the flow attributes.
5485  * @param[in] items
5486  *   Pointer to the list of items.
5487  * @param[in] actions
5488  *   Pointer to the list of actions.
5489  * @param[in] external
5490  *   This flow rule is created by request external to PMD.
5491  * @param[in] hairpin
5492  *   Number of hairpin TX actions, 0 means classic flow.
5493  * @param[out] error
5494  *   Pointer to the error structure.
5495  *
5496  * @return
5497  *   0 on success, a negative errno value otherwise and rte_errno is set.
5498  */
5499 static int
5500 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
5501                  const struct rte_flow_item items[],
5502                  const struct rte_flow_action actions[],
5503                  bool external, int hairpin, struct rte_flow_error *error)
5504 {
5505         int ret;
5506         uint64_t action_flags = 0;
5507         uint64_t item_flags = 0;
5508         uint64_t last_item = 0;
5509         uint8_t next_protocol = 0xff;
5510         uint16_t ether_type = 0;
5511         int actions_n = 0;
5512         uint8_t item_ipv6_proto = 0;
5513         const struct rte_flow_item *gre_item = NULL;
5514         const struct rte_flow_action_raw_decap *decap;
5515         const struct rte_flow_action_raw_encap *encap;
5516         const struct rte_flow_action_rss *rss;
5517         const struct rte_flow_item_tcp nic_tcp_mask = {
5518                 .hdr = {
5519                         .tcp_flags = 0xFF,
5520                         .src_port = RTE_BE16(UINT16_MAX),
5521                         .dst_port = RTE_BE16(UINT16_MAX),
5522                 }
5523         };
5524         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
5525                 .hdr = {
5526                         .src_addr =
5527                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5528                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5529                         .dst_addr =
5530                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5531                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5532                         .vtc_flow = RTE_BE32(0xffffffff),
5533                         .proto = 0xff,
5534                         .hop_limits = 0xff,
5535                 },
5536                 .has_frag_ext = 1,
5537         };
5538         const struct rte_flow_item_ecpri nic_ecpri_mask = {
5539                 .hdr = {
5540                         .common = {
5541                                 .u32 =
5542                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
5543                                         .type = 0xFF,
5544                                         }).u32),
5545                         },
5546                         .dummy[0] = 0xffffffff,
5547                 },
5548         };
5549         struct mlx5_priv *priv = dev->data->dev_private;
5550         struct mlx5_dev_config *dev_conf = &priv->config;
5551         uint16_t queue_index = 0xFFFF;
5552         const struct rte_flow_item_vlan *vlan_m = NULL;
5553         int16_t rw_act_num = 0;
5554         uint64_t is_root;
5555
5556         if (items == NULL)
5557                 return -1;
5558         ret = flow_dv_validate_attributes(dev, attr, external, error);
5559         if (ret < 0)
5560                 return ret;
5561         is_root = (uint64_t)ret;
5562         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5563                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5564                 int type = items->type;
5565
5566                 if (!mlx5_flow_os_item_supported(type))
5567                         return rte_flow_error_set(error, ENOTSUP,
5568                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5569                                                   NULL, "item not supported");
5570                 switch (type) {
5571                 case RTE_FLOW_ITEM_TYPE_VOID:
5572                         break;
5573                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5574                         ret = flow_dv_validate_item_port_id
5575                                         (dev, items, attr, item_flags, error);
5576                         if (ret < 0)
5577                                 return ret;
5578                         last_item = MLX5_FLOW_ITEM_PORT_ID;
5579                         break;
5580                 case RTE_FLOW_ITEM_TYPE_ETH:
5581                         ret = mlx5_flow_validate_item_eth(items, item_flags,
5582                                                           error);
5583                         if (ret < 0)
5584                                 return ret;
5585                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5586                                              MLX5_FLOW_LAYER_OUTER_L2;
5587                         if (items->mask != NULL && items->spec != NULL) {
5588                                 ether_type =
5589                                         ((const struct rte_flow_item_eth *)
5590                                          items->spec)->type;
5591                                 ether_type &=
5592                                         ((const struct rte_flow_item_eth *)
5593                                          items->mask)->type;
5594                                 ether_type = rte_be_to_cpu_16(ether_type);
5595                         } else {
5596                                 ether_type = 0;
5597                         }
5598                         break;
5599                 case RTE_FLOW_ITEM_TYPE_VLAN:
5600                         ret = flow_dv_validate_item_vlan(items, item_flags,
5601                                                          dev, error);
5602                         if (ret < 0)
5603                                 return ret;
5604                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5605                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5606                         if (items->mask != NULL && items->spec != NULL) {
5607                                 ether_type =
5608                                         ((const struct rte_flow_item_vlan *)
5609                                          items->spec)->inner_type;
5610                                 ether_type &=
5611                                         ((const struct rte_flow_item_vlan *)
5612                                          items->mask)->inner_type;
5613                                 ether_type = rte_be_to_cpu_16(ether_type);
5614                         } else {
5615                                 ether_type = 0;
5616                         }
5617                         /* Store outer VLAN mask for of_push_vlan action. */
5618                         if (!tunnel)
5619                                 vlan_m = items->mask;
5620                         break;
5621                 case RTE_FLOW_ITEM_TYPE_IPV4:
5622                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5623                                                   &item_flags, &tunnel);
5624                         ret = flow_dv_validate_item_ipv4(items, item_flags,
5625                                                          last_item, ether_type,
5626                                                          error);
5627                         if (ret < 0)
5628                                 return ret;
5629                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5630                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5631                         if (items->mask != NULL &&
5632                             ((const struct rte_flow_item_ipv4 *)
5633                              items->mask)->hdr.next_proto_id) {
5634                                 next_protocol =
5635                                         ((const struct rte_flow_item_ipv4 *)
5636                                          (items->spec))->hdr.next_proto_id;
5637                                 next_protocol &=
5638                                         ((const struct rte_flow_item_ipv4 *)
5639                                          (items->mask))->hdr.next_proto_id;
5640                         } else {
5641                                 /* Reset for inner layer. */
5642                                 next_protocol = 0xff;
5643                         }
5644                         break;
5645                 case RTE_FLOW_ITEM_TYPE_IPV6:
5646                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5647                                                   &item_flags, &tunnel);
5648                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5649                                                            last_item,
5650                                                            ether_type,
5651                                                            &nic_ipv6_mask,
5652                                                            error);
5653                         if (ret < 0)
5654                                 return ret;
5655                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5656                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5657                         if (items->mask != NULL &&
5658                             ((const struct rte_flow_item_ipv6 *)
5659                              items->mask)->hdr.proto) {
5660                                 item_ipv6_proto =
5661                                         ((const struct rte_flow_item_ipv6 *)
5662                                          items->spec)->hdr.proto;
5663                                 next_protocol =
5664                                         ((const struct rte_flow_item_ipv6 *)
5665                                          items->spec)->hdr.proto;
5666                                 next_protocol &=
5667                                         ((const struct rte_flow_item_ipv6 *)
5668                                          items->mask)->hdr.proto;
5669                         } else {
5670                                 /* Reset for inner layer. */
5671                                 next_protocol = 0xff;
5672                         }
5673                         break;
5674                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
5675                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
5676                                                                   item_flags,
5677                                                                   error);
5678                         if (ret < 0)
5679                                 return ret;
5680                         last_item = tunnel ?
5681                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
5682                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
5683                         if (items->mask != NULL &&
5684                             ((const struct rte_flow_item_ipv6_frag_ext *)
5685                              items->mask)->hdr.next_header) {
5686                                 next_protocol =
5687                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5688                                  items->spec)->hdr.next_header;
5689                                 next_protocol &=
5690                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5691                                  items->mask)->hdr.next_header;
5692                         } else {
5693                                 /* Reset for inner layer. */
5694                                 next_protocol = 0xff;
5695                         }
5696                         break;
5697                 case RTE_FLOW_ITEM_TYPE_TCP:
5698                         ret = mlx5_flow_validate_item_tcp
5699                                                 (items, item_flags,
5700                                                  next_protocol,
5701                                                  &nic_tcp_mask,
5702                                                  error);
5703                         if (ret < 0)
5704                                 return ret;
5705                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5706                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5707                         break;
5708                 case RTE_FLOW_ITEM_TYPE_UDP:
5709                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5710                                                           next_protocol,
5711                                                           error);
5712                         if (ret < 0)
5713                                 return ret;
5714                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5715                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5716                         break;
5717                 case RTE_FLOW_ITEM_TYPE_GRE:
5718                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5719                                                           next_protocol, error);
5720                         if (ret < 0)
5721                                 return ret;
5722                         gre_item = items;
5723                         last_item = MLX5_FLOW_LAYER_GRE;
5724                         break;
5725                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5726                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5727                                                             next_protocol,
5728                                                             error);
5729                         if (ret < 0)
5730                                 return ret;
5731                         last_item = MLX5_FLOW_LAYER_NVGRE;
5732                         break;
5733                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5734                         ret = mlx5_flow_validate_item_gre_key
5735                                 (items, item_flags, gre_item, error);
5736                         if (ret < 0)
5737                                 return ret;
5738                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5739                         break;
5740                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5741                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5742                                                             error);
5743                         if (ret < 0)
5744                                 return ret;
5745                         last_item = MLX5_FLOW_LAYER_VXLAN;
5746                         break;
5747                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5748                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5749                                                                 item_flags, dev,
5750                                                                 error);
5751                         if (ret < 0)
5752                                 return ret;
5753                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5754                         break;
5755                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5756                         ret = mlx5_flow_validate_item_geneve(items,
5757                                                              item_flags, dev,
5758                                                              error);
5759                         if (ret < 0)
5760                                 return ret;
5761                         last_item = MLX5_FLOW_LAYER_GENEVE;
5762                         break;
5763                 case RTE_FLOW_ITEM_TYPE_MPLS:
5764                         ret = mlx5_flow_validate_item_mpls(dev, items,
5765                                                            item_flags,
5766                                                            last_item, error);
5767                         if (ret < 0)
5768                                 return ret;
5769                         last_item = MLX5_FLOW_LAYER_MPLS;
5770                         break;
5771
5772                 case RTE_FLOW_ITEM_TYPE_MARK:
5773                         ret = flow_dv_validate_item_mark(dev, items, attr,
5774                                                          error);
5775                         if (ret < 0)
5776                                 return ret;
5777                         last_item = MLX5_FLOW_ITEM_MARK;
5778                         break;
5779                 case RTE_FLOW_ITEM_TYPE_META:
5780                         ret = flow_dv_validate_item_meta(dev, items, attr,
5781                                                          error);
5782                         if (ret < 0)
5783                                 return ret;
5784                         last_item = MLX5_FLOW_ITEM_METADATA;
5785                         break;
5786                 case RTE_FLOW_ITEM_TYPE_ICMP:
5787                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5788                                                            next_protocol,
5789                                                            error);
5790                         if (ret < 0)
5791                                 return ret;
5792                         last_item = MLX5_FLOW_LAYER_ICMP;
5793                         break;
5794                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5795                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5796                                                             next_protocol,
5797                                                             error);
5798                         if (ret < 0)
5799                                 return ret;
5800                         item_ipv6_proto = IPPROTO_ICMPV6;
5801                         last_item = MLX5_FLOW_LAYER_ICMP6;
5802                         break;
5803                 case RTE_FLOW_ITEM_TYPE_TAG:
5804                         ret = flow_dv_validate_item_tag(dev, items,
5805                                                         attr, error);
5806                         if (ret < 0)
5807                                 return ret;
5808                         last_item = MLX5_FLOW_ITEM_TAG;
5809                         break;
5810                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5811                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5812                         break;
5813                 case RTE_FLOW_ITEM_TYPE_GTP:
5814                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5815                                                         error);
5816                         if (ret < 0)
5817                                 return ret;
5818                         last_item = MLX5_FLOW_LAYER_GTP;
5819                         break;
5820                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5821                         /* Capacity will be checked in the translate stage. */
5822                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5823                                                             last_item,
5824                                                             ether_type,
5825                                                             &nic_ecpri_mask,
5826                                                             error);
5827                         if (ret < 0)
5828                                 return ret;
5829                         last_item = MLX5_FLOW_LAYER_ECPRI;
5830                         break;
5831                 default:
5832                         return rte_flow_error_set(error, ENOTSUP,
5833                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5834                                                   NULL, "item not supported");
5835                 }
5836                 item_flags |= last_item;
5837         }
5838         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5839                 int type = actions->type;
5840
5841                 if (!mlx5_flow_os_action_supported(type))
5842                         return rte_flow_error_set(error, ENOTSUP,
5843                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5844                                                   actions,
5845                                                   "action not supported");
5846                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5847                         return rte_flow_error_set(error, ENOTSUP,
5848                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5849                                                   actions, "too many actions");
5850                 switch (type) {
5851                 case RTE_FLOW_ACTION_TYPE_VOID:
5852                         break;
5853                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5854                         ret = flow_dv_validate_action_port_id(dev,
5855                                                               action_flags,
5856                                                               actions,
5857                                                               attr,
5858                                                               error);
5859                         if (ret)
5860                                 return ret;
5861                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5862                         ++actions_n;
5863                         break;
5864                 case RTE_FLOW_ACTION_TYPE_FLAG:
5865                         ret = flow_dv_validate_action_flag(dev, action_flags,
5866                                                            attr, error);
5867                         if (ret < 0)
5868                                 return ret;
5869                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5870                                 /* Count all modify-header actions as one. */
5871                                 if (!(action_flags &
5872                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5873                                         ++actions_n;
5874                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5875                                                 MLX5_FLOW_ACTION_MARK_EXT;
5876                         } else {
5877                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5878                                 ++actions_n;
5879                         }
5880                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5881                         break;
5882                 case RTE_FLOW_ACTION_TYPE_MARK:
5883                         ret = flow_dv_validate_action_mark(dev, actions,
5884                                                            action_flags,
5885                                                            attr, error);
5886                         if (ret < 0)
5887                                 return ret;
5888                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5889                                 /* Count all modify-header actions as one. */
5890                                 if (!(action_flags &
5891                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5892                                         ++actions_n;
5893                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5894                                                 MLX5_FLOW_ACTION_MARK_EXT;
5895                         } else {
5896                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5897                                 ++actions_n;
5898                         }
5899                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5900                         break;
5901                 case RTE_FLOW_ACTION_TYPE_SET_META:
5902                         ret = flow_dv_validate_action_set_meta(dev, actions,
5903                                                                action_flags,
5904                                                                attr, error);
5905                         if (ret < 0)
5906                                 return ret;
5907                         /* Count all modify-header actions as one action. */
5908                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5909                                 ++actions_n;
5910                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5911                         rw_act_num += MLX5_ACT_NUM_SET_META;
5912                         break;
5913                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5914                         ret = flow_dv_validate_action_set_tag(dev, actions,
5915                                                               action_flags,
5916                                                               attr, error);
5917                         if (ret < 0)
5918                                 return ret;
5919                         /* Count all modify-header actions as one action. */
5920                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5921                                 ++actions_n;
5922                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5923                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5924                         break;
5925                 case RTE_FLOW_ACTION_TYPE_DROP:
5926                         ret = mlx5_flow_validate_action_drop(action_flags,
5927                                                              attr, error);
5928                         if (ret < 0)
5929                                 return ret;
5930                         action_flags |= MLX5_FLOW_ACTION_DROP;
5931                         ++actions_n;
5932                         break;
5933                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5934                         ret = mlx5_flow_validate_action_queue(actions,
5935                                                               action_flags, dev,
5936                                                               attr, error);
5937                         if (ret < 0)
5938                                 return ret;
5939                         queue_index = ((const struct rte_flow_action_queue *)
5940                                                         (actions->conf))->index;
5941                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5942                         ++actions_n;
5943                         break;
5944                 case RTE_FLOW_ACTION_TYPE_RSS:
5945                         rss = actions->conf;
5946                         ret = mlx5_flow_validate_action_rss(actions,
5947                                                             action_flags, dev,
5948                                                             attr, item_flags,
5949                                                             error);
5950                         if (ret < 0)
5951                                 return ret;
5952                         if (rss != NULL && rss->queue_num)
5953                                 queue_index = rss->queue[0];
5954                         action_flags |= MLX5_FLOW_ACTION_RSS;
5955                         ++actions_n;
5956                         break;
5957                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5958                         ret =
5959                         mlx5_flow_validate_action_default_miss(action_flags,
5960                                         attr, error);
5961                         if (ret < 0)
5962                                 return ret;
5963                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5964                         ++actions_n;
5965                         break;
5966                 case RTE_FLOW_ACTION_TYPE_COUNT:
5967                         ret = flow_dv_validate_action_count(dev, error);
5968                         if (ret < 0)
5969                                 return ret;
5970                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5971                         ++actions_n;
5972                         break;
5973                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5974                         if (flow_dv_validate_action_pop_vlan(dev,
5975                                                              action_flags,
5976                                                              actions,
5977                                                              item_flags, attr,
5978                                                              error))
5979                                 return -rte_errno;
5980                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5981                         ++actions_n;
5982                         break;
5983                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5984                         ret = flow_dv_validate_action_push_vlan(dev,
5985                                                                 action_flags,
5986                                                                 vlan_m,
5987                                                                 actions, attr,
5988                                                                 error);
5989                         if (ret < 0)
5990                                 return ret;
5991                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5992                         ++actions_n;
5993                         break;
5994                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5995                         ret = flow_dv_validate_action_set_vlan_pcp
5996                                                 (action_flags, actions, error);
5997                         if (ret < 0)
5998                                 return ret;
5999                         /* Count PCP with push_vlan command. */
6000                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
6001                         break;
6002                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
6003                         ret = flow_dv_validate_action_set_vlan_vid
6004                                                 (item_flags, action_flags,
6005                                                  actions, error);
6006                         if (ret < 0)
6007                                 return ret;
6008                         /* Count VID with push_vlan command. */
6009                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
6010                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
6011                         break;
6012                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
6013                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
6014                         ret = flow_dv_validate_action_l2_encap(dev,
6015                                                                action_flags,
6016                                                                actions, attr,
6017                                                                error);
6018                         if (ret < 0)
6019                                 return ret;
6020                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
6021                         ++actions_n;
6022                         break;
6023                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
6024                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
6025                         ret = flow_dv_validate_action_decap(dev, action_flags,
6026                                                             attr, error);
6027                         if (ret < 0)
6028                                 return ret;
6029                         action_flags |= MLX5_FLOW_ACTION_DECAP;
6030                         ++actions_n;
6031                         break;
6032                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
6033                         ret = flow_dv_validate_action_raw_encap_decap
6034                                 (dev, NULL, actions->conf, attr, &action_flags,
6035                                  &actions_n, error);
6036                         if (ret < 0)
6037                                 return ret;
6038                         break;
6039                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
6040                         decap = actions->conf;
6041                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
6042                                 ;
6043                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
6044                                 encap = NULL;
6045                                 actions--;
6046                         } else {
6047                                 encap = actions->conf;
6048                         }
6049                         ret = flow_dv_validate_action_raw_encap_decap
6050                                            (dev,
6051                                             decap ? decap : &empty_decap, encap,
6052                                             attr, &action_flags, &actions_n,
6053                                             error);
6054                         if (ret < 0)
6055                                 return ret;
6056                         break;
6057                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
6058                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
6059                         ret = flow_dv_validate_action_modify_mac(action_flags,
6060                                                                  actions,
6061                                                                  item_flags,
6062                                                                  error);
6063                         if (ret < 0)
6064                                 return ret;
6065                         /* Count all modify-header actions as one action. */
6066                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6067                                 ++actions_n;
6068                         action_flags |= actions->type ==
6069                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
6070                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
6071                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
6072                         /*
6073                          * Even if the source and destination MAC addresses have
6074                          * overlap in the header with 4B alignment, the convert
6075                          * function will handle them separately and 4 SW actions
6076                          * will be created. And 2 actions will be added each
6077                          * time no matter how many bytes of address will be set.
6078                          */
6079                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
6080                         break;
6081                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
6082                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
6083                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
6084                                                                   actions,
6085                                                                   item_flags,
6086                                                                   error);
6087                         if (ret < 0)
6088                                 return ret;
6089                         /* Count all modify-header actions as one action. */
6090                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6091                                 ++actions_n;
6092                         action_flags |= actions->type ==
6093                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
6094                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
6095                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
6096                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
6097                         break;
6098                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
6099                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
6100                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
6101                                                                   actions,
6102                                                                   item_flags,
6103                                                                   error);
6104                         if (ret < 0)
6105                                 return ret;
6106                         if (item_ipv6_proto == IPPROTO_ICMPV6)
6107                                 return rte_flow_error_set(error, ENOTSUP,
6108                                         RTE_FLOW_ERROR_TYPE_ACTION,
6109                                         actions,
6110                                         "Can't change header "
6111                                         "with ICMPv6 proto");
6112                         /* Count all modify-header actions as one action. */
6113                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6114                                 ++actions_n;
6115                         action_flags |= actions->type ==
6116                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
6117                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
6118                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
6119                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
6120                         break;
6121                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
6122                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
6123                         ret = flow_dv_validate_action_modify_tp(action_flags,
6124                                                                 actions,
6125                                                                 item_flags,
6126                                                                 error);
6127                         if (ret < 0)
6128                                 return ret;
6129                         /* Count all modify-header actions as one action. */
6130                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6131                                 ++actions_n;
6132                         action_flags |= actions->type ==
6133                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
6134                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
6135                                                 MLX5_FLOW_ACTION_SET_TP_DST;
6136                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
6137                         break;
6138                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
6139                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
6140                         ret = flow_dv_validate_action_modify_ttl(action_flags,
6141                                                                  actions,
6142                                                                  item_flags,
6143                                                                  error);
6144                         if (ret < 0)
6145                                 return ret;
6146                         /* Count all modify-header actions as one action. */
6147                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6148                                 ++actions_n;
6149                         action_flags |= actions->type ==
6150                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
6151                                                 MLX5_FLOW_ACTION_SET_TTL :
6152                                                 MLX5_FLOW_ACTION_DEC_TTL;
6153                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
6154                         break;
6155                 case RTE_FLOW_ACTION_TYPE_JUMP:
6156                         ret = flow_dv_validate_action_jump(actions,
6157                                                            action_flags,
6158                                                            attr, external,
6159                                                            error);
6160                         if (ret)
6161                                 return ret;
6162                         ++actions_n;
6163                         action_flags |= MLX5_FLOW_ACTION_JUMP;
6164                         break;
6165                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
6166                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
6167                         ret = flow_dv_validate_action_modify_tcp_seq
6168                                                                 (action_flags,
6169                                                                  actions,
6170                                                                  item_flags,
6171                                                                  error);
6172                         if (ret < 0)
6173                                 return ret;
6174                         /* Count all modify-header actions as one action. */
6175                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6176                                 ++actions_n;
6177                         action_flags |= actions->type ==
6178                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
6179                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
6180                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
6181                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
6182                         break;
6183                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
6184                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
6185                         ret = flow_dv_validate_action_modify_tcp_ack
6186                                                                 (action_flags,
6187                                                                  actions,
6188                                                                  item_flags,
6189                                                                  error);
6190                         if (ret < 0)
6191                                 return ret;
6192                         /* Count all modify-header actions as one action. */
6193                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6194                                 ++actions_n;
6195                         action_flags |= actions->type ==
6196                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
6197                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
6198                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
6199                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
6200                         break;
6201                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
6202                         break;
6203                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
6204                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
6205                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
6206                         break;
6207                 case RTE_FLOW_ACTION_TYPE_METER:
6208                         ret = mlx5_flow_validate_action_meter(dev,
6209                                                               action_flags,
6210                                                               actions, attr,
6211                                                               error);
6212                         if (ret < 0)
6213                                 return ret;
6214                         action_flags |= MLX5_FLOW_ACTION_METER;
6215                         ++actions_n;
6216                         /* Meter action will add one more TAG action. */
6217                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
6218                         break;
6219                 case RTE_FLOW_ACTION_TYPE_AGE:
6220                         ret = flow_dv_validate_action_age(action_flags,
6221                                                           actions, dev,
6222                                                           error);
6223                         if (ret < 0)
6224                                 return ret;
6225                         action_flags |= MLX5_FLOW_ACTION_AGE;
6226                         ++actions_n;
6227                         break;
6228                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
6229                         ret = flow_dv_validate_action_modify_ipv4_dscp
6230                                                          (action_flags,
6231                                                           actions,
6232                                                           item_flags,
6233                                                           error);
6234                         if (ret < 0)
6235                                 return ret;
6236                         /* Count all modify-header actions as one action. */
6237                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6238                                 ++actions_n;
6239                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
6240                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6241                         break;
6242                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
6243                         ret = flow_dv_validate_action_modify_ipv6_dscp
6244                                                                 (action_flags,
6245                                                                  actions,
6246                                                                  item_flags,
6247                                                                  error);
6248                         if (ret < 0)
6249                                 return ret;
6250                         /* Count all modify-header actions as one action. */
6251                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6252                                 ++actions_n;
6253                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
6254                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6255                         break;
6256                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
6257                         ret = flow_dv_validate_action_sample(action_flags,
6258                                                              actions, dev,
6259                                                              attr, error);
6260                         if (ret < 0)
6261                                 return ret;
6262                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
6263                         ++actions_n;
6264                         break;
6265                 default:
6266                         return rte_flow_error_set(error, ENOTSUP,
6267                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6268                                                   actions,
6269                                                   "action not supported");
6270                 }
6271         }
6272         /*
6273          * Validate the drop action mutual exclusion with other actions.
6274          * Drop action is mutually-exclusive with any other action, except for
6275          * Count action.
6276          */
6277         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
6278             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
6279                 return rte_flow_error_set(error, EINVAL,
6280                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6281                                           "Drop action is mutually-exclusive "
6282                                           "with any other action, except for "
6283                                           "Count action");
6284         /* Eswitch has few restrictions on using items and actions */
6285         if (attr->transfer) {
6286                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6287                     action_flags & MLX5_FLOW_ACTION_FLAG)
6288                         return rte_flow_error_set(error, ENOTSUP,
6289                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6290                                                   NULL,
6291                                                   "unsupported action FLAG");
6292                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6293                     action_flags & MLX5_FLOW_ACTION_MARK)
6294                         return rte_flow_error_set(error, ENOTSUP,
6295                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6296                                                   NULL,
6297                                                   "unsupported action MARK");
6298                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
6299                         return rte_flow_error_set(error, ENOTSUP,
6300                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6301                                                   NULL,
6302                                                   "unsupported action QUEUE");
6303                 if (action_flags & MLX5_FLOW_ACTION_RSS)
6304                         return rte_flow_error_set(error, ENOTSUP,
6305                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6306                                                   NULL,
6307                                                   "unsupported action RSS");
6308                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
6309                         return rte_flow_error_set(error, EINVAL,
6310                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6311                                                   actions,
6312                                                   "no fate action is found");
6313         } else {
6314                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
6315                         return rte_flow_error_set(error, EINVAL,
6316                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6317                                                   actions,
6318                                                   "no fate action is found");
6319         }
6320         /* Continue validation for Xcap and VLAN actions.*/
6321         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
6322                              MLX5_FLOW_VLAN_ACTIONS)) &&
6323             (queue_index == 0xFFFF ||
6324              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
6325                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
6326                     MLX5_FLOW_XCAP_ACTIONS)
6327                         return rte_flow_error_set(error, ENOTSUP,
6328                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6329                                                   NULL, "encap and decap "
6330                                                   "combination aren't supported");
6331                 if (!attr->transfer && attr->ingress) {
6332                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
6333                                 return rte_flow_error_set
6334                                                 (error, ENOTSUP,
6335                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6336                                                  NULL, "encap is not supported"
6337                                                  " for ingress traffic");
6338                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6339                                 return rte_flow_error_set
6340                                                 (error, ENOTSUP,
6341                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6342                                                  NULL, "push VLAN action not "
6343                                                  "supported for ingress");
6344                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
6345                                         MLX5_FLOW_VLAN_ACTIONS)
6346                                 return rte_flow_error_set
6347                                                 (error, ENOTSUP,
6348                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6349                                                  NULL, "no support for "
6350                                                  "multiple VLAN actions");
6351                 }
6352         }
6353         /* Hairpin flow will add one more TAG action. */
6354         if (hairpin > 0)
6355                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6356         /* extra metadata enabled: one more TAG action will be add. */
6357         if (dev_conf->dv_flow_en &&
6358             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
6359             mlx5_flow_ext_mreg_supported(dev))
6360                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6361         if ((uint32_t)rw_act_num >
6362                         flow_dv_modify_hdr_action_max(dev, is_root)) {
6363                 return rte_flow_error_set(error, ENOTSUP,
6364                                           RTE_FLOW_ERROR_TYPE_ACTION,
6365                                           NULL, "too many header modify"
6366                                           " actions to support");
6367         }
6368         return 0;
6369 }
6370
6371 /**
6372  * Internal preparation function. Allocates the DV flow size,
6373  * this size is constant.
6374  *
6375  * @param[in] dev
6376  *   Pointer to the rte_eth_dev structure.
6377  * @param[in] attr
6378  *   Pointer to the flow attributes.
6379  * @param[in] items
6380  *   Pointer to the list of items.
6381  * @param[in] actions
6382  *   Pointer to the list of actions.
6383  * @param[out] error
6384  *   Pointer to the error structure.
6385  *
6386  * @return
6387  *   Pointer to mlx5_flow object on success,
6388  *   otherwise NULL and rte_errno is set.
6389  */
6390 static struct mlx5_flow *
6391 flow_dv_prepare(struct rte_eth_dev *dev,
6392                 const struct rte_flow_attr *attr __rte_unused,
6393                 const struct rte_flow_item items[] __rte_unused,
6394                 const struct rte_flow_action actions[] __rte_unused,
6395                 struct rte_flow_error *error)
6396 {
6397         uint32_t handle_idx = 0;
6398         struct mlx5_flow *dev_flow;
6399         struct mlx5_flow_handle *dev_handle;
6400         struct mlx5_priv *priv = dev->data->dev_private;
6401
6402         /* In case of corrupting the memory. */
6403         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
6404                 rte_flow_error_set(error, ENOSPC,
6405                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6406                                    "not free temporary device flow");
6407                 return NULL;
6408         }
6409         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
6410                                    &handle_idx);
6411         if (!dev_handle) {
6412                 rte_flow_error_set(error, ENOMEM,
6413                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6414                                    "not enough memory to create flow handle");
6415                 return NULL;
6416         }
6417         /* No multi-thread supporting. */
6418         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
6419         dev_flow->handle = dev_handle;
6420         dev_flow->handle_idx = handle_idx;
6421         /*
6422          * In some old rdma-core releases, before continuing, a check of the
6423          * length of matching parameter will be done at first. It needs to use
6424          * the length without misc4 param. If the flow has misc4 support, then
6425          * the length needs to be adjusted accordingly. Each param member is
6426          * aligned with a 64B boundary naturally.
6427          */
6428         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
6429                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
6430         /*
6431          * The matching value needs to be cleared to 0 before using. In the
6432          * past, it will be automatically cleared when using rte_*alloc
6433          * API. The time consumption will be almost the same as before.
6434          */
6435         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
6436         dev_flow->ingress = attr->ingress;
6437         dev_flow->dv.transfer = attr->transfer;
6438         return dev_flow;
6439 }
6440
6441 #ifdef RTE_LIBRTE_MLX5_DEBUG
6442 /**
6443  * Sanity check for match mask and value. Similar to check_valid_spec() in
6444  * kernel driver. If unmasked bit is present in value, it returns failure.
6445  *
6446  * @param match_mask
6447  *   pointer to match mask buffer.
6448  * @param match_value
6449  *   pointer to match value buffer.
6450  *
6451  * @return
6452  *   0 if valid, -EINVAL otherwise.
6453  */
6454 static int
6455 flow_dv_check_valid_spec(void *match_mask, void *match_value)
6456 {
6457         uint8_t *m = match_mask;
6458         uint8_t *v = match_value;
6459         unsigned int i;
6460
6461         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
6462                 if (v[i] & ~m[i]) {
6463                         DRV_LOG(ERR,
6464                                 "match_value differs from match_criteria"
6465                                 " %p[%u] != %p[%u]",
6466                                 match_value, i, match_mask, i);
6467                         return -EINVAL;
6468                 }
6469         }
6470         return 0;
6471 }
6472 #endif
6473
6474 /**
6475  * Add match of ip_version.
6476  *
6477  * @param[in] group
6478  *   Flow group.
6479  * @param[in] headers_v
6480  *   Values header pointer.
6481  * @param[in] headers_m
6482  *   Masks header pointer.
6483  * @param[in] ip_version
6484  *   The IP version to set.
6485  */
6486 static inline void
6487 flow_dv_set_match_ip_version(uint32_t group,
6488                              void *headers_v,
6489                              void *headers_m,
6490                              uint8_t ip_version)
6491 {
6492         if (group == 0)
6493                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
6494         else
6495                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
6496                          ip_version);
6497         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
6498         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
6499         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
6500 }
6501
6502 /**
6503  * Add Ethernet item to matcher and to the value.
6504  *
6505  * @param[in, out] matcher
6506  *   Flow matcher.
6507  * @param[in, out] key
6508  *   Flow matcher value.
6509  * @param[in] item
6510  *   Flow pattern to translate.
6511  * @param[in] inner
6512  *   Item is inner pattern.
6513  */
6514 static void
6515 flow_dv_translate_item_eth(void *matcher, void *key,
6516                            const struct rte_flow_item *item, int inner,
6517                            uint32_t group)
6518 {
6519         const struct rte_flow_item_eth *eth_m = item->mask;
6520         const struct rte_flow_item_eth *eth_v = item->spec;
6521         const struct rte_flow_item_eth nic_mask = {
6522                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6523                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6524                 .type = RTE_BE16(0xffff),
6525         };
6526         void *headers_m;
6527         void *headers_v;
6528         char *l24_v;
6529         unsigned int i;
6530
6531         if (!eth_v)
6532                 return;
6533         if (!eth_m)
6534                 eth_m = &nic_mask;
6535         if (inner) {
6536                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6537                                          inner_headers);
6538                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6539         } else {
6540                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6541                                          outer_headers);
6542                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6543         }
6544         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
6545                &eth_m->dst, sizeof(eth_m->dst));
6546         /* The value must be in the range of the mask. */
6547         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
6548         for (i = 0; i < sizeof(eth_m->dst); ++i)
6549                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
6550         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
6551                &eth_m->src, sizeof(eth_m->src));
6552         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
6553         /* The value must be in the range of the mask. */
6554         for (i = 0; i < sizeof(eth_m->dst); ++i)
6555                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
6556         if (eth_v->type) {
6557                 /* When ethertype is present set mask for tagged VLAN. */
6558                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6559                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
6560                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
6561                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
6562                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
6563                                  1);
6564                         /* Return here to avoid setting match on ethertype. */
6565                         return;
6566                 }
6567         }
6568         /*
6569          * HW supports match on one Ethertype, the Ethertype following the last
6570          * VLAN tag of the packet (see PRM).
6571          * Set match on ethertype only if ETH header is not followed by VLAN.
6572          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6573          * ethertype, and use ip_version field instead.
6574          * eCPRI over Ether layer will use type value 0xAEFE.
6575          */
6576         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6577             eth_m->type == 0xFFFF) {
6578                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6579         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6580                    eth_m->type == 0xFFFF) {
6581                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6582         } else {
6583                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6584                          rte_be_to_cpu_16(eth_m->type));
6585                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6586                                      ethertype);
6587                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
6588         }
6589 }
6590
6591 /**
6592  * Add VLAN item to matcher and to the value.
6593  *
6594  * @param[in, out] dev_flow
6595  *   Flow descriptor.
6596  * @param[in, out] matcher
6597  *   Flow matcher.
6598  * @param[in, out] key
6599  *   Flow matcher value.
6600  * @param[in] item
6601  *   Flow pattern to translate.
6602  * @param[in] inner
6603  *   Item is inner pattern.
6604  */
6605 static void
6606 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
6607                             void *matcher, void *key,
6608                             const struct rte_flow_item *item,
6609                             int inner, uint32_t group)
6610 {
6611         const struct rte_flow_item_vlan *vlan_m = item->mask;
6612         const struct rte_flow_item_vlan *vlan_v = item->spec;
6613         void *headers_m;
6614         void *headers_v;
6615         uint16_t tci_m;
6616         uint16_t tci_v;
6617
6618         if (inner) {
6619                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6620                                          inner_headers);
6621                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6622         } else {
6623                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6624                                          outer_headers);
6625                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6626                 /*
6627                  * This is workaround, masks are not supported,
6628                  * and pre-validated.
6629                  */
6630                 if (vlan_v)
6631                         dev_flow->handle->vf_vlan.tag =
6632                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
6633         }
6634         /*
6635          * When VLAN item exists in flow, mark packet as tagged,
6636          * even if TCI is not specified.
6637          */
6638         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6639         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
6640         if (!vlan_v)
6641                 return;
6642         if (!vlan_m)
6643                 vlan_m = &rte_flow_item_vlan_mask;
6644         tci_m = rte_be_to_cpu_16(vlan_m->tci);
6645         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6646         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
6647         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
6648         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
6649         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
6650         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6651         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6652         /*
6653          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6654          * ethertype, and use ip_version field instead.
6655          */
6656         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6657             vlan_m->inner_type == 0xFFFF) {
6658                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6659         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6660                    vlan_m->inner_type == 0xFFFF) {
6661                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6662         } else {
6663                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6664                          rte_be_to_cpu_16(vlan_m->inner_type));
6665                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6666                          rte_be_to_cpu_16(vlan_m->inner_type &
6667                                           vlan_v->inner_type));
6668         }
6669 }
6670
6671 /**
6672  * Add IPV4 item to matcher and to the value.
6673  *
6674  * @param[in, out] matcher
6675  *   Flow matcher.
6676  * @param[in, out] key
6677  *   Flow matcher value.
6678  * @param[in] item
6679  *   Flow pattern to translate.
6680  * @param[in] item_flags
6681  *   Bit-fields that holds the items detected until now.
6682  * @param[in] inner
6683  *   Item is inner pattern.
6684  * @param[in] group
6685  *   The group to insert the rule.
6686  */
6687 static void
6688 flow_dv_translate_item_ipv4(void *matcher, void *key,
6689                             const struct rte_flow_item *item,
6690                             const uint64_t item_flags,
6691                             int inner, uint32_t group)
6692 {
6693         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6694         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6695         const struct rte_flow_item_ipv4 nic_mask = {
6696                 .hdr = {
6697                         .src_addr = RTE_BE32(0xffffffff),
6698                         .dst_addr = RTE_BE32(0xffffffff),
6699                         .type_of_service = 0xff,
6700                         .next_proto_id = 0xff,
6701                         .time_to_live = 0xff,
6702                 },
6703         };
6704         void *headers_m;
6705         void *headers_v;
6706         char *l24_m;
6707         char *l24_v;
6708         uint8_t tos;
6709
6710         if (inner) {
6711                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6712                                          inner_headers);
6713                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6714         } else {
6715                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6716                                          outer_headers);
6717                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6718         }
6719         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6720         /*
6721          * On outer header (which must contains L2), or inner header with L2,
6722          * set cvlan_tag mask bit to mark this packet as untagged.
6723          * This should be done even if item->spec is empty.
6724          */
6725         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6726                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6727         if (!ipv4_v)
6728                 return;
6729         if (!ipv4_m)
6730                 ipv4_m = &nic_mask;
6731         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6732                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6733         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6734                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6735         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6736         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6737         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6738                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6739         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6740                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6741         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6742         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6743         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6744         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6745                  ipv4_m->hdr.type_of_service);
6746         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6747         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6748                  ipv4_m->hdr.type_of_service >> 2);
6749         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6750         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6751                  ipv4_m->hdr.next_proto_id);
6752         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6753                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6754         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6755                  ipv4_m->hdr.time_to_live);
6756         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6757                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6758         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6759                  !!(ipv4_m->hdr.fragment_offset));
6760         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6761                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
6762 }
6763
6764 /**
6765  * Add IPV6 item to matcher and to the value.
6766  *
6767  * @param[in, out] matcher
6768  *   Flow matcher.
6769  * @param[in, out] key
6770  *   Flow matcher value.
6771  * @param[in] item
6772  *   Flow pattern to translate.
6773  * @param[in] item_flags
6774  *   Bit-fields that holds the items detected until now.
6775  * @param[in] inner
6776  *   Item is inner pattern.
6777  * @param[in] group
6778  *   The group to insert the rule.
6779  */
6780 static void
6781 flow_dv_translate_item_ipv6(void *matcher, void *key,
6782                             const struct rte_flow_item *item,
6783                             const uint64_t item_flags,
6784                             int inner, uint32_t group)
6785 {
6786         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6787         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6788         const struct rte_flow_item_ipv6 nic_mask = {
6789                 .hdr = {
6790                         .src_addr =
6791                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6792                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6793                         .dst_addr =
6794                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6795                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6796                         .vtc_flow = RTE_BE32(0xffffffff),
6797                         .proto = 0xff,
6798                         .hop_limits = 0xff,
6799                 },
6800         };
6801         void *headers_m;
6802         void *headers_v;
6803         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6804         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6805         char *l24_m;
6806         char *l24_v;
6807         uint32_t vtc_m;
6808         uint32_t vtc_v;
6809         int i;
6810         int size;
6811
6812         if (inner) {
6813                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6814                                          inner_headers);
6815                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6816         } else {
6817                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6818                                          outer_headers);
6819                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6820         }
6821         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6822         /*
6823          * On outer header (which must contains L2), or inner header with L2,
6824          * set cvlan_tag mask bit to mark this packet as untagged.
6825          * This should be done even if item->spec is empty.
6826          */
6827         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6828                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6829         if (!ipv6_v)
6830                 return;
6831         if (!ipv6_m)
6832                 ipv6_m = &nic_mask;
6833         size = sizeof(ipv6_m->hdr.dst_addr);
6834         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6835                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6836         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6837                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6838         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6839         for (i = 0; i < size; ++i)
6840                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6841         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6842                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6843         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6844                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6845         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6846         for (i = 0; i < size; ++i)
6847                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6848         /* TOS. */
6849         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6850         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6851         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6852         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6853         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6854         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6855         /* Label. */
6856         if (inner) {
6857                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6858                          vtc_m);
6859                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6860                          vtc_v);
6861         } else {
6862                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6863                          vtc_m);
6864                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6865                          vtc_v);
6866         }
6867         /* Protocol. */
6868         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6869                  ipv6_m->hdr.proto);
6870         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6871                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6872         /* Hop limit. */
6873         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6874                  ipv6_m->hdr.hop_limits);
6875         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6876                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6877         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6878                  !!(ipv6_m->has_frag_ext));
6879         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6880                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
6881 }
6882
6883 /**
6884  * Add IPV6 fragment extension item to matcher and to the value.
6885  *
6886  * @param[in, out] matcher
6887  *   Flow matcher.
6888  * @param[in, out] key
6889  *   Flow matcher value.
6890  * @param[in] item
6891  *   Flow pattern to translate.
6892  * @param[in] inner
6893  *   Item is inner pattern.
6894  */
6895 static void
6896 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
6897                                      const struct rte_flow_item *item,
6898                                      int inner)
6899 {
6900         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
6901         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
6902         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
6903                 .hdr = {
6904                         .next_header = 0xff,
6905                         .frag_data = RTE_BE16(0xffff),
6906                 },
6907         };
6908         void *headers_m;
6909         void *headers_v;
6910
6911         if (inner) {
6912                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6913                                          inner_headers);
6914                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6915         } else {
6916                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6917                                          outer_headers);
6918                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6919         }
6920         /* IPv6 fragment extension item exists, so packet is IP fragment. */
6921         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
6922         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
6923         if (!ipv6_frag_ext_v)
6924                 return;
6925         if (!ipv6_frag_ext_m)
6926                 ipv6_frag_ext_m = &nic_mask;
6927         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6928                  ipv6_frag_ext_m->hdr.next_header);
6929         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6930                  ipv6_frag_ext_v->hdr.next_header &
6931                  ipv6_frag_ext_m->hdr.next_header);
6932 }
6933
6934 /**
6935  * Add TCP item to matcher and to the value.
6936  *
6937  * @param[in, out] matcher
6938  *   Flow matcher.
6939  * @param[in, out] key
6940  *   Flow matcher value.
6941  * @param[in] item
6942  *   Flow pattern to translate.
6943  * @param[in] inner
6944  *   Item is inner pattern.
6945  */
6946 static void
6947 flow_dv_translate_item_tcp(void *matcher, void *key,
6948                            const struct rte_flow_item *item,
6949                            int inner)
6950 {
6951         const struct rte_flow_item_tcp *tcp_m = item->mask;
6952         const struct rte_flow_item_tcp *tcp_v = item->spec;
6953         void *headers_m;
6954         void *headers_v;
6955
6956         if (inner) {
6957                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6958                                          inner_headers);
6959                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6960         } else {
6961                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6962                                          outer_headers);
6963                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6964         }
6965         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6966         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6967         if (!tcp_v)
6968                 return;
6969         if (!tcp_m)
6970                 tcp_m = &rte_flow_item_tcp_mask;
6971         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6972                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6973         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6974                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6975         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6976                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6977         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6978                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6979         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6980                  tcp_m->hdr.tcp_flags);
6981         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6982                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6983 }
6984
6985 /**
6986  * Add UDP item to matcher and to the value.
6987  *
6988  * @param[in, out] matcher
6989  *   Flow matcher.
6990  * @param[in, out] key
6991  *   Flow matcher value.
6992  * @param[in] item
6993  *   Flow pattern to translate.
6994  * @param[in] inner
6995  *   Item is inner pattern.
6996  */
6997 static void
6998 flow_dv_translate_item_udp(void *matcher, void *key,
6999                            const struct rte_flow_item *item,
7000                            int inner)
7001 {
7002         const struct rte_flow_item_udp *udp_m = item->mask;
7003         const struct rte_flow_item_udp *udp_v = item->spec;
7004         void *headers_m;
7005         void *headers_v;
7006
7007         if (inner) {
7008                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7009                                          inner_headers);
7010                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7011         } else {
7012                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7013                                          outer_headers);
7014                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7015         }
7016         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7017         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
7018         if (!udp_v)
7019                 return;
7020         if (!udp_m)
7021                 udp_m = &rte_flow_item_udp_mask;
7022         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
7023                  rte_be_to_cpu_16(udp_m->hdr.src_port));
7024         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
7025                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
7026         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
7027                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
7028         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7029                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
7030 }
7031
7032 /**
7033  * Add GRE optional Key item to matcher and to the value.
7034  *
7035  * @param[in, out] matcher
7036  *   Flow matcher.
7037  * @param[in, out] key
7038  *   Flow matcher value.
7039  * @param[in] item
7040  *   Flow pattern to translate.
7041  * @param[in] inner
7042  *   Item is inner pattern.
7043  */
7044 static void
7045 flow_dv_translate_item_gre_key(void *matcher, void *key,
7046                                    const struct rte_flow_item *item)
7047 {
7048         const rte_be32_t *key_m = item->mask;
7049         const rte_be32_t *key_v = item->spec;
7050         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7051         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7052         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
7053
7054         /* GRE K bit must be on and should already be validated */
7055         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
7056         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
7057         if (!key_v)
7058                 return;
7059         if (!key_m)
7060                 key_m = &gre_key_default_mask;
7061         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
7062                  rte_be_to_cpu_32(*key_m) >> 8);
7063         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
7064                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
7065         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
7066                  rte_be_to_cpu_32(*key_m) & 0xFF);
7067         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
7068                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
7069 }
7070
7071 /**
7072  * Add GRE item to matcher and to the value.
7073  *
7074  * @param[in, out] matcher
7075  *   Flow matcher.
7076  * @param[in, out] key
7077  *   Flow matcher value.
7078  * @param[in] item
7079  *   Flow pattern to translate.
7080  * @param[in] inner
7081  *   Item is inner pattern.
7082  */
7083 static void
7084 flow_dv_translate_item_gre(void *matcher, void *key,
7085                            const struct rte_flow_item *item,
7086                            int inner)
7087 {
7088         const struct rte_flow_item_gre *gre_m = item->mask;
7089         const struct rte_flow_item_gre *gre_v = item->spec;
7090         void *headers_m;
7091         void *headers_v;
7092         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7093         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7094         struct {
7095                 union {
7096                         __extension__
7097                         struct {
7098                                 uint16_t version:3;
7099                                 uint16_t rsvd0:9;
7100                                 uint16_t s_present:1;
7101                                 uint16_t k_present:1;
7102                                 uint16_t rsvd_bit1:1;
7103                                 uint16_t c_present:1;
7104                         };
7105                         uint16_t value;
7106                 };
7107         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
7108
7109         if (inner) {
7110                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7111                                          inner_headers);
7112                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7113         } else {
7114                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7115                                          outer_headers);
7116                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7117         }
7118         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7119         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
7120         if (!gre_v)
7121                 return;
7122         if (!gre_m)
7123                 gre_m = &rte_flow_item_gre_mask;
7124         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
7125                  rte_be_to_cpu_16(gre_m->protocol));
7126         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7127                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
7128         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
7129         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
7130         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
7131                  gre_crks_rsvd0_ver_m.c_present);
7132         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
7133                  gre_crks_rsvd0_ver_v.c_present &
7134                  gre_crks_rsvd0_ver_m.c_present);
7135         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
7136                  gre_crks_rsvd0_ver_m.k_present);
7137         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
7138                  gre_crks_rsvd0_ver_v.k_present &
7139                  gre_crks_rsvd0_ver_m.k_present);
7140         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
7141                  gre_crks_rsvd0_ver_m.s_present);
7142         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
7143                  gre_crks_rsvd0_ver_v.s_present &
7144                  gre_crks_rsvd0_ver_m.s_present);
7145 }
7146
7147 /**
7148  * Add NVGRE item to matcher and to the value.
7149  *
7150  * @param[in, out] matcher
7151  *   Flow matcher.
7152  * @param[in, out] key
7153  *   Flow matcher value.
7154  * @param[in] item
7155  *   Flow pattern to translate.
7156  * @param[in] inner
7157  *   Item is inner pattern.
7158  */
7159 static void
7160 flow_dv_translate_item_nvgre(void *matcher, void *key,
7161                              const struct rte_flow_item *item,
7162                              int inner)
7163 {
7164         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
7165         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
7166         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7167         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7168         const char *tni_flow_id_m;
7169         const char *tni_flow_id_v;
7170         char *gre_key_m;
7171         char *gre_key_v;
7172         int size;
7173         int i;
7174
7175         /* For NVGRE, GRE header fields must be set with defined values. */
7176         const struct rte_flow_item_gre gre_spec = {
7177                 .c_rsvd0_ver = RTE_BE16(0x2000),
7178                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
7179         };
7180         const struct rte_flow_item_gre gre_mask = {
7181                 .c_rsvd0_ver = RTE_BE16(0xB000),
7182                 .protocol = RTE_BE16(UINT16_MAX),
7183         };
7184         const struct rte_flow_item gre_item = {
7185                 .spec = &gre_spec,
7186                 .mask = &gre_mask,
7187                 .last = NULL,
7188         };
7189         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
7190         if (!nvgre_v)
7191                 return;
7192         if (!nvgre_m)
7193                 nvgre_m = &rte_flow_item_nvgre_mask;
7194         tni_flow_id_m = (const char *)nvgre_m->tni;
7195         tni_flow_id_v = (const char *)nvgre_v->tni;
7196         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
7197         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
7198         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
7199         memcpy(gre_key_m, tni_flow_id_m, size);
7200         for (i = 0; i < size; ++i)
7201                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
7202 }
7203
7204 /**
7205  * Add VXLAN item to matcher and to the value.
7206  *
7207  * @param[in, out] matcher
7208  *   Flow matcher.
7209  * @param[in, out] key
7210  *   Flow matcher value.
7211  * @param[in] item
7212  *   Flow pattern to translate.
7213  * @param[in] inner
7214  *   Item is inner pattern.
7215  */
7216 static void
7217 flow_dv_translate_item_vxlan(void *matcher, void *key,
7218                              const struct rte_flow_item *item,
7219                              int inner)
7220 {
7221         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
7222         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
7223         void *headers_m;
7224         void *headers_v;
7225         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7226         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7227         char *vni_m;
7228         char *vni_v;
7229         uint16_t dport;
7230         int size;
7231         int i;
7232
7233         if (inner) {
7234                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7235                                          inner_headers);
7236                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7237         } else {
7238                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7239                                          outer_headers);
7240                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7241         }
7242         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7243                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7244         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7245                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7246                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7247         }
7248         if (!vxlan_v)
7249                 return;
7250         if (!vxlan_m)
7251                 vxlan_m = &rte_flow_item_vxlan_mask;
7252         size = sizeof(vxlan_m->vni);
7253         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
7254         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
7255         memcpy(vni_m, vxlan_m->vni, size);
7256         for (i = 0; i < size; ++i)
7257                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7258 }
7259
7260 /**
7261  * Add VXLAN-GPE item to matcher and to the value.
7262  *
7263  * @param[in, out] matcher
7264  *   Flow matcher.
7265  * @param[in, out] key
7266  *   Flow matcher value.
7267  * @param[in] item
7268  *   Flow pattern to translate.
7269  * @param[in] inner
7270  *   Item is inner pattern.
7271  */
7272
7273 static void
7274 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
7275                                  const struct rte_flow_item *item, int inner)
7276 {
7277         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
7278         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
7279         void *headers_m;
7280         void *headers_v;
7281         void *misc_m =
7282                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
7283         void *misc_v =
7284                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7285         char *vni_m;
7286         char *vni_v;
7287         uint16_t dport;
7288         int size;
7289         int i;
7290         uint8_t flags_m = 0xff;
7291         uint8_t flags_v = 0xc;
7292
7293         if (inner) {
7294                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7295                                          inner_headers);
7296                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7297         } else {
7298                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7299                                          outer_headers);
7300                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7301         }
7302         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7303                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7304         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7305                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7306                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7307         }
7308         if (!vxlan_v)
7309                 return;
7310         if (!vxlan_m)
7311                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
7312         size = sizeof(vxlan_m->vni);
7313         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
7314         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
7315         memcpy(vni_m, vxlan_m->vni, size);
7316         for (i = 0; i < size; ++i)
7317                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7318         if (vxlan_m->flags) {
7319                 flags_m = vxlan_m->flags;
7320                 flags_v = vxlan_v->flags;
7321         }
7322         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
7323         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
7324         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
7325                  vxlan_m->protocol);
7326         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
7327                  vxlan_v->protocol);
7328 }
7329
7330 /**
7331  * Add Geneve item to matcher and to the value.
7332  *
7333  * @param[in, out] matcher
7334  *   Flow matcher.
7335  * @param[in, out] key
7336  *   Flow matcher value.
7337  * @param[in] item
7338  *   Flow pattern to translate.
7339  * @param[in] inner
7340  *   Item is inner pattern.
7341  */
7342
7343 static void
7344 flow_dv_translate_item_geneve(void *matcher, void *key,
7345                               const struct rte_flow_item *item, int inner)
7346 {
7347         const struct rte_flow_item_geneve *geneve_m = item->mask;
7348         const struct rte_flow_item_geneve *geneve_v = item->spec;
7349         void *headers_m;
7350         void *headers_v;
7351         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7352         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7353         uint16_t dport;
7354         uint16_t gbhdr_m;
7355         uint16_t gbhdr_v;
7356         char *vni_m;
7357         char *vni_v;
7358         size_t size, i;
7359
7360         if (inner) {
7361                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7362                                          inner_headers);
7363                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7364         } else {
7365                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7366                                          outer_headers);
7367                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7368         }
7369         dport = MLX5_UDP_PORT_GENEVE;
7370         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7371                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7372                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7373         }
7374         if (!geneve_v)
7375                 return;
7376         if (!geneve_m)
7377                 geneve_m = &rte_flow_item_geneve_mask;
7378         size = sizeof(geneve_m->vni);
7379         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
7380         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
7381         memcpy(vni_m, geneve_m->vni, size);
7382         for (i = 0; i < size; ++i)
7383                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
7384         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
7385                  rte_be_to_cpu_16(geneve_m->protocol));
7386         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
7387                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
7388         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
7389         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
7390         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
7391                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7392         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
7393                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7394         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
7395                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7396         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
7397                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
7398                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7399 }
7400
7401 /**
7402  * Add MPLS item to matcher and to the value.
7403  *
7404  * @param[in, out] matcher
7405  *   Flow matcher.
7406  * @param[in, out] key
7407  *   Flow matcher value.
7408  * @param[in] item
7409  *   Flow pattern to translate.
7410  * @param[in] prev_layer
7411  *   The protocol layer indicated in previous item.
7412  * @param[in] inner
7413  *   Item is inner pattern.
7414  */
7415 static void
7416 flow_dv_translate_item_mpls(void *matcher, void *key,
7417                             const struct rte_flow_item *item,
7418                             uint64_t prev_layer,
7419                             int inner)
7420 {
7421         const uint32_t *in_mpls_m = item->mask;
7422         const uint32_t *in_mpls_v = item->spec;
7423         uint32_t *out_mpls_m = 0;
7424         uint32_t *out_mpls_v = 0;
7425         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7426         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7427         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
7428                                      misc_parameters_2);
7429         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7430         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
7431         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7432
7433         switch (prev_layer) {
7434         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7435                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
7436                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7437                          MLX5_UDP_PORT_MPLS);
7438                 break;
7439         case MLX5_FLOW_LAYER_GRE:
7440                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
7441                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7442                          RTE_ETHER_TYPE_MPLS);
7443                 break;
7444         default:
7445                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7446                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
7447                          IPPROTO_MPLS);
7448                 break;
7449         }
7450         if (!in_mpls_v)
7451                 return;
7452         if (!in_mpls_m)
7453                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
7454         switch (prev_layer) {
7455         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7456                 out_mpls_m =
7457                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7458                                                  outer_first_mpls_over_udp);
7459                 out_mpls_v =
7460                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7461                                                  outer_first_mpls_over_udp);
7462                 break;
7463         case MLX5_FLOW_LAYER_GRE:
7464                 out_mpls_m =
7465                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7466                                                  outer_first_mpls_over_gre);
7467                 out_mpls_v =
7468                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7469                                                  outer_first_mpls_over_gre);
7470                 break;
7471         default:
7472                 /* Inner MPLS not over GRE is not supported. */
7473                 if (!inner) {
7474                         out_mpls_m =
7475                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7476                                                          misc2_m,
7477                                                          outer_first_mpls);
7478                         out_mpls_v =
7479                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7480                                                          misc2_v,
7481                                                          outer_first_mpls);
7482                 }
7483                 break;
7484         }
7485         if (out_mpls_m && out_mpls_v) {
7486                 *out_mpls_m = *in_mpls_m;
7487                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
7488         }
7489 }
7490
7491 /**
7492  * Add metadata register item to matcher
7493  *
7494  * @param[in, out] matcher
7495  *   Flow matcher.
7496  * @param[in, out] key
7497  *   Flow matcher value.
7498  * @param[in] reg_type
7499  *   Type of device metadata register
7500  * @param[in] value
7501  *   Register value
7502  * @param[in] mask
7503  *   Register mask
7504  */
7505 static void
7506 flow_dv_match_meta_reg(void *matcher, void *key,
7507                        enum modify_reg reg_type,
7508                        uint32_t data, uint32_t mask)
7509 {
7510         void *misc2_m =
7511                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
7512         void *misc2_v =
7513                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7514         uint32_t temp;
7515
7516         data &= mask;
7517         switch (reg_type) {
7518         case REG_A:
7519                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
7520                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
7521                 break;
7522         case REG_B:
7523                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
7524                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
7525                 break;
7526         case REG_C_0:
7527                 /*
7528                  * The metadata register C0 field might be divided into
7529                  * source vport index and META item value, we should set
7530                  * this field according to specified mask, not as whole one.
7531                  */
7532                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
7533                 temp |= mask;
7534                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
7535                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
7536                 temp &= ~mask;
7537                 temp |= data;
7538                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
7539                 break;
7540         case REG_C_1:
7541                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
7542                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
7543                 break;
7544         case REG_C_2:
7545                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
7546                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
7547                 break;
7548         case REG_C_3:
7549                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
7550                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
7551                 break;
7552         case REG_C_4:
7553                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
7554                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
7555                 break;
7556         case REG_C_5:
7557                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
7558                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
7559                 break;
7560         case REG_C_6:
7561                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
7562                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
7563                 break;
7564         case REG_C_7:
7565                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
7566                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
7567                 break;
7568         default:
7569                 MLX5_ASSERT(false);
7570                 break;
7571         }
7572 }
7573
7574 /**
7575  * Add MARK item to matcher
7576  *
7577  * @param[in] dev
7578  *   The device to configure through.
7579  * @param[in, out] matcher
7580  *   Flow matcher.
7581  * @param[in, out] key
7582  *   Flow matcher value.
7583  * @param[in] item
7584  *   Flow pattern to translate.
7585  */
7586 static void
7587 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
7588                             void *matcher, void *key,
7589                             const struct rte_flow_item *item)
7590 {
7591         struct mlx5_priv *priv = dev->data->dev_private;
7592         const struct rte_flow_item_mark *mark;
7593         uint32_t value;
7594         uint32_t mask;
7595
7596         mark = item->mask ? (const void *)item->mask :
7597                             &rte_flow_item_mark_mask;
7598         mask = mark->id & priv->sh->dv_mark_mask;
7599         mark = (const void *)item->spec;
7600         MLX5_ASSERT(mark);
7601         value = mark->id & priv->sh->dv_mark_mask & mask;
7602         if (mask) {
7603                 enum modify_reg reg;
7604
7605                 /* Get the metadata register index for the mark. */
7606                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
7607                 MLX5_ASSERT(reg > 0);
7608                 if (reg == REG_C_0) {
7609                         struct mlx5_priv *priv = dev->data->dev_private;
7610                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7611                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7612
7613                         mask &= msk_c0;
7614                         mask <<= shl_c0;
7615                         value <<= shl_c0;
7616                 }
7617                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7618         }
7619 }
7620
7621 /**
7622  * Add META item to matcher
7623  *
7624  * @param[in] dev
7625  *   The devich to configure through.
7626  * @param[in, out] matcher
7627  *   Flow matcher.
7628  * @param[in, out] key
7629  *   Flow matcher value.
7630  * @param[in] attr
7631  *   Attributes of flow that includes this item.
7632  * @param[in] item
7633  *   Flow pattern to translate.
7634  */
7635 static void
7636 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
7637                             void *matcher, void *key,
7638                             const struct rte_flow_attr *attr,
7639                             const struct rte_flow_item *item)
7640 {
7641         const struct rte_flow_item_meta *meta_m;
7642         const struct rte_flow_item_meta *meta_v;
7643
7644         meta_m = (const void *)item->mask;
7645         if (!meta_m)
7646                 meta_m = &rte_flow_item_meta_mask;
7647         meta_v = (const void *)item->spec;
7648         if (meta_v) {
7649                 int reg;
7650                 uint32_t value = meta_v->data;
7651                 uint32_t mask = meta_m->data;
7652
7653                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
7654                 if (reg < 0)
7655                         return;
7656                 /*
7657                  * In datapath code there is no endianness
7658                  * coversions for perfromance reasons, all
7659                  * pattern conversions are done in rte_flow.
7660                  */
7661                 value = rte_cpu_to_be_32(value);
7662                 mask = rte_cpu_to_be_32(mask);
7663                 if (reg == REG_C_0) {
7664                         struct mlx5_priv *priv = dev->data->dev_private;
7665                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7666                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7667 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7668                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
7669
7670                         value >>= shr_c0;
7671                         mask >>= shr_c0;
7672 #endif
7673                         value <<= shl_c0;
7674                         mask <<= shl_c0;
7675                         MLX5_ASSERT(msk_c0);
7676                         MLX5_ASSERT(!(~msk_c0 & mask));
7677                 }
7678                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7679         }
7680 }
7681
7682 /**
7683  * Add vport metadata Reg C0 item to matcher
7684  *
7685  * @param[in, out] matcher
7686  *   Flow matcher.
7687  * @param[in, out] key
7688  *   Flow matcher value.
7689  * @param[in] reg
7690  *   Flow pattern to translate.
7691  */
7692 static void
7693 flow_dv_translate_item_meta_vport(void *matcher, void *key,
7694                                   uint32_t value, uint32_t mask)
7695 {
7696         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
7697 }
7698
7699 /**
7700  * Add tag item to matcher
7701  *
7702  * @param[in] dev
7703  *   The devich to configure through.
7704  * @param[in, out] matcher
7705  *   Flow matcher.
7706  * @param[in, out] key
7707  *   Flow matcher value.
7708  * @param[in] item
7709  *   Flow pattern to translate.
7710  */
7711 static void
7712 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7713                                 void *matcher, void *key,
7714                                 const struct rte_flow_item *item)
7715 {
7716         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7717         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7718         uint32_t mask, value;
7719
7720         MLX5_ASSERT(tag_v);
7721         value = tag_v->data;
7722         mask = tag_m ? tag_m->data : UINT32_MAX;
7723         if (tag_v->id == REG_C_0) {
7724                 struct mlx5_priv *priv = dev->data->dev_private;
7725                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7726                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7727
7728                 mask &= msk_c0;
7729                 mask <<= shl_c0;
7730                 value <<= shl_c0;
7731         }
7732         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7733 }
7734
7735 /**
7736  * Add TAG item to matcher
7737  *
7738  * @param[in] dev
7739  *   The devich to configure through.
7740  * @param[in, out] matcher
7741  *   Flow matcher.
7742  * @param[in, out] key
7743  *   Flow matcher value.
7744  * @param[in] item
7745  *   Flow pattern to translate.
7746  */
7747 static void
7748 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7749                            void *matcher, void *key,
7750                            const struct rte_flow_item *item)
7751 {
7752         const struct rte_flow_item_tag *tag_v = item->spec;
7753         const struct rte_flow_item_tag *tag_m = item->mask;
7754         enum modify_reg reg;
7755
7756         MLX5_ASSERT(tag_v);
7757         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7758         /* Get the metadata register index for the tag. */
7759         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7760         MLX5_ASSERT(reg > 0);
7761         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7762 }
7763
7764 /**
7765  * Add source vport match to the specified matcher.
7766  *
7767  * @param[in, out] matcher
7768  *   Flow matcher.
7769  * @param[in, out] key
7770  *   Flow matcher value.
7771  * @param[in] port
7772  *   Source vport value to match
7773  * @param[in] mask
7774  *   Mask
7775  */
7776 static void
7777 flow_dv_translate_item_source_vport(void *matcher, void *key,
7778                                     int16_t port, uint16_t mask)
7779 {
7780         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7781         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7782
7783         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7784         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7785 }
7786
7787 /**
7788  * Translate port-id item to eswitch match on  port-id.
7789  *
7790  * @param[in] dev
7791  *   The devich to configure through.
7792  * @param[in, out] matcher
7793  *   Flow matcher.
7794  * @param[in, out] key
7795  *   Flow matcher value.
7796  * @param[in] item
7797  *   Flow pattern to translate.
7798  *
7799  * @return
7800  *   0 on success, a negative errno value otherwise.
7801  */
7802 static int
7803 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7804                                void *key, const struct rte_flow_item *item)
7805 {
7806         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7807         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7808         struct mlx5_priv *priv;
7809         uint16_t mask, id;
7810
7811         mask = pid_m ? pid_m->id : 0xffff;
7812         id = pid_v ? pid_v->id : dev->data->port_id;
7813         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7814         if (!priv)
7815                 return -rte_errno;
7816         /* Translate to vport field or to metadata, depending on mode. */
7817         if (priv->vport_meta_mask)
7818                 flow_dv_translate_item_meta_vport(matcher, key,
7819                                                   priv->vport_meta_tag,
7820                                                   priv->vport_meta_mask);
7821         else
7822                 flow_dv_translate_item_source_vport(matcher, key,
7823                                                     priv->vport_id, mask);
7824         return 0;
7825 }
7826
7827 /**
7828  * Add ICMP6 item to matcher and to the value.
7829  *
7830  * @param[in, out] matcher
7831  *   Flow matcher.
7832  * @param[in, out] key
7833  *   Flow matcher value.
7834  * @param[in] item
7835  *   Flow pattern to translate.
7836  * @param[in] inner
7837  *   Item is inner pattern.
7838  */
7839 static void
7840 flow_dv_translate_item_icmp6(void *matcher, void *key,
7841                               const struct rte_flow_item *item,
7842                               int inner)
7843 {
7844         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7845         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7846         void *headers_m;
7847         void *headers_v;
7848         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7849                                      misc_parameters_3);
7850         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7851         if (inner) {
7852                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7853                                          inner_headers);
7854                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7855         } else {
7856                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7857                                          outer_headers);
7858                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7859         }
7860         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7861         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7862         if (!icmp6_v)
7863                 return;
7864         if (!icmp6_m)
7865                 icmp6_m = &rte_flow_item_icmp6_mask;
7866         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7867         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7868                  icmp6_v->type & icmp6_m->type);
7869         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7870         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7871                  icmp6_v->code & icmp6_m->code);
7872 }
7873
7874 /**
7875  * Add ICMP item to matcher and to the value.
7876  *
7877  * @param[in, out] matcher
7878  *   Flow matcher.
7879  * @param[in, out] key
7880  *   Flow matcher value.
7881  * @param[in] item
7882  *   Flow pattern to translate.
7883  * @param[in] inner
7884  *   Item is inner pattern.
7885  */
7886 static void
7887 flow_dv_translate_item_icmp(void *matcher, void *key,
7888                             const struct rte_flow_item *item,
7889                             int inner)
7890 {
7891         const struct rte_flow_item_icmp *icmp_m = item->mask;
7892         const struct rte_flow_item_icmp *icmp_v = item->spec;
7893         uint32_t icmp_header_data_m = 0;
7894         uint32_t icmp_header_data_v = 0;
7895         void *headers_m;
7896         void *headers_v;
7897         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7898                                      misc_parameters_3);
7899         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7900         if (inner) {
7901                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7902                                          inner_headers);
7903                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7904         } else {
7905                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7906                                          outer_headers);
7907                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7908         }
7909         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7910         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7911         if (!icmp_v)
7912                 return;
7913         if (!icmp_m)
7914                 icmp_m = &rte_flow_item_icmp_mask;
7915         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7916                  icmp_m->hdr.icmp_type);
7917         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7918                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7919         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7920                  icmp_m->hdr.icmp_code);
7921         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7922                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7923         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
7924         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
7925         if (icmp_header_data_m) {
7926                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
7927                 icmp_header_data_v |=
7928                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
7929                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
7930                          icmp_header_data_m);
7931                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
7932                          icmp_header_data_v & icmp_header_data_m);
7933         }
7934 }
7935
7936 /**
7937  * Add GTP item to matcher and to the value.
7938  *
7939  * @param[in, out] matcher
7940  *   Flow matcher.
7941  * @param[in, out] key
7942  *   Flow matcher value.
7943  * @param[in] item
7944  *   Flow pattern to translate.
7945  * @param[in] inner
7946  *   Item is inner pattern.
7947  */
7948 static void
7949 flow_dv_translate_item_gtp(void *matcher, void *key,
7950                            const struct rte_flow_item *item, int inner)
7951 {
7952         const struct rte_flow_item_gtp *gtp_m = item->mask;
7953         const struct rte_flow_item_gtp *gtp_v = item->spec;
7954         void *headers_m;
7955         void *headers_v;
7956         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7957                                      misc_parameters_3);
7958         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7959         uint16_t dport = RTE_GTPU_UDP_PORT;
7960
7961         if (inner) {
7962                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7963                                          inner_headers);
7964                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7965         } else {
7966                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7967                                          outer_headers);
7968                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7969         }
7970         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7971                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7972                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7973         }
7974         if (!gtp_v)
7975                 return;
7976         if (!gtp_m)
7977                 gtp_m = &rte_flow_item_gtp_mask;
7978         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7979                  gtp_m->v_pt_rsv_flags);
7980         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7981                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7982         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7983         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7984                  gtp_v->msg_type & gtp_m->msg_type);
7985         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7986                  rte_be_to_cpu_32(gtp_m->teid));
7987         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7988                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7989 }
7990
7991 /**
7992  * Add eCPRI item to matcher and to the value.
7993  *
7994  * @param[in] dev
7995  *   The devich to configure through.
7996  * @param[in, out] matcher
7997  *   Flow matcher.
7998  * @param[in, out] key
7999  *   Flow matcher value.
8000  * @param[in] item
8001  *   Flow pattern to translate.
8002  * @param[in] samples
8003  *   Sample IDs to be used in the matching.
8004  */
8005 static void
8006 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
8007                              void *key, const struct rte_flow_item *item)
8008 {
8009         struct mlx5_priv *priv = dev->data->dev_private;
8010         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
8011         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
8012         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
8013                                      misc_parameters_4);
8014         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
8015         uint32_t *samples;
8016         void *dw_m;
8017         void *dw_v;
8018
8019         if (!ecpri_v)
8020                 return;
8021         if (!ecpri_m)
8022                 ecpri_m = &rte_flow_item_ecpri_mask;
8023         /*
8024          * Maximal four DW samples are supported in a single matching now.
8025          * Two are used now for a eCPRI matching:
8026          * 1. Type: one byte, mask should be 0x00ff0000 in network order
8027          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
8028          *    if any.
8029          */
8030         if (!ecpri_m->hdr.common.u32)
8031                 return;
8032         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
8033         /* Need to take the whole DW as the mask to fill the entry. */
8034         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
8035                             prog_sample_field_value_0);
8036         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
8037                             prog_sample_field_value_0);
8038         /* Already big endian (network order) in the header. */
8039         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
8040         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
8041         /* Sample#0, used for matching type, offset 0. */
8042         MLX5_SET(fte_match_set_misc4, misc4_m,
8043                  prog_sample_field_id_0, samples[0]);
8044         /* It makes no sense to set the sample ID in the mask field. */
8045         MLX5_SET(fte_match_set_misc4, misc4_v,
8046                  prog_sample_field_id_0, samples[0]);
8047         /*
8048          * Checking if message body part needs to be matched.
8049          * Some wildcard rules only matching type field should be supported.
8050          */
8051         if (ecpri_m->hdr.dummy[0]) {
8052                 switch (ecpri_v->hdr.common.type) {
8053                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
8054                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
8055                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
8056                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
8057                                             prog_sample_field_value_1);
8058                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
8059                                             prog_sample_field_value_1);
8060                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
8061                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
8062                         /* Sample#1, to match message body, offset 4. */
8063                         MLX5_SET(fte_match_set_misc4, misc4_m,
8064                                  prog_sample_field_id_1, samples[1]);
8065                         MLX5_SET(fte_match_set_misc4, misc4_v,
8066                                  prog_sample_field_id_1, samples[1]);
8067                         break;
8068                 default:
8069                         /* Others, do not match any sample ID. */
8070                         break;
8071                 }
8072         }
8073 }
8074
8075 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
8076
8077 #define HEADER_IS_ZERO(match_criteria, headers)                              \
8078         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
8079                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
8080
8081 /**
8082  * Calculate flow matcher enable bitmap.
8083  *
8084  * @param match_criteria
8085  *   Pointer to flow matcher criteria.
8086  *
8087  * @return
8088  *   Bitmap of enabled fields.
8089  */
8090 static uint8_t
8091 flow_dv_matcher_enable(uint32_t *match_criteria)
8092 {
8093         uint8_t match_criteria_enable;
8094
8095         match_criteria_enable =
8096                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
8097                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
8098         match_criteria_enable |=
8099                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
8100                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
8101         match_criteria_enable |=
8102                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
8103                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
8104         match_criteria_enable |=
8105                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
8106                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
8107         match_criteria_enable |=
8108                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
8109                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
8110         match_criteria_enable |=
8111                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
8112                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
8113         return match_criteria_enable;
8114 }
8115
8116
8117 /**
8118  * Get a flow table.
8119  *
8120  * @param[in, out] dev
8121  *   Pointer to rte_eth_dev structure.
8122  * @param[in] table_id
8123  *   Table id to use.
8124  * @param[in] egress
8125  *   Direction of the table.
8126  * @param[in] transfer
8127  *   E-Switch or NIC flow.
8128  * @param[out] error
8129  *   pointer to error structure.
8130  *
8131  * @return
8132  *   Returns tables resource based on the index, NULL in case of failed.
8133  */
8134 static struct mlx5_flow_tbl_resource *
8135 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
8136                          uint32_t table_id, uint8_t egress,
8137                          uint8_t transfer,
8138                          struct rte_flow_error *error)
8139 {
8140         struct mlx5_priv *priv = dev->data->dev_private;
8141         struct mlx5_dev_ctx_shared *sh = priv->sh;
8142         struct mlx5_flow_tbl_resource *tbl;
8143         union mlx5_flow_tbl_key table_key = {
8144                 {
8145                         .table_id = table_id,
8146                         .reserved = 0,
8147                         .domain = !!transfer,
8148                         .direction = !!egress,
8149                 }
8150         };
8151         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
8152                                                          table_key.v64);
8153         struct mlx5_flow_tbl_data_entry *tbl_data;
8154         uint32_t idx = 0;
8155         int ret;
8156         void *domain;
8157
8158         if (pos) {
8159                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
8160                                         entry);
8161                 tbl = &tbl_data->tbl;
8162                 rte_atomic32_inc(&tbl->refcnt);
8163                 return tbl;
8164         }
8165         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
8166         if (!tbl_data) {
8167                 rte_flow_error_set(error, ENOMEM,
8168                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8169                                    NULL,
8170                                    "cannot allocate flow table data entry");
8171                 return NULL;
8172         }
8173         tbl_data->idx = idx;
8174         tbl = &tbl_data->tbl;
8175         pos = &tbl_data->entry;
8176         if (transfer)
8177                 domain = sh->fdb_domain;
8178         else if (egress)
8179                 domain = sh->tx_domain;
8180         else
8181                 domain = sh->rx_domain;
8182         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
8183         if (ret) {
8184                 rte_flow_error_set(error, ENOMEM,
8185                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8186                                    NULL, "cannot create flow table object");
8187                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
8188                 return NULL;
8189         }
8190         /*
8191          * No multi-threads now, but still better to initialize the reference
8192          * count before insert it into the hash list.
8193          */
8194         rte_atomic32_init(&tbl->refcnt);
8195         /* Jump action reference count is initialized here. */
8196         rte_atomic32_init(&tbl_data->jump.refcnt);
8197         pos->key = table_key.v64;
8198         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
8199         if (ret < 0) {
8200                 rte_flow_error_set(error, -ret,
8201                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8202                                    "cannot insert flow table data entry");
8203                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8204                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
8205         }
8206         rte_atomic32_inc(&tbl->refcnt);
8207         return tbl;
8208 }
8209
8210 /**
8211  * Release a flow table.
8212  *
8213  * @param[in] dev
8214  *   Pointer to rte_eth_dev structure.
8215  * @param[in] tbl
8216  *   Table resource to be released.
8217  *
8218  * @return
8219  *   Returns 0 if table was released, else return 1;
8220  */
8221 static int
8222 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
8223                              struct mlx5_flow_tbl_resource *tbl)
8224 {
8225         struct mlx5_priv *priv = dev->data->dev_private;
8226         struct mlx5_dev_ctx_shared *sh = priv->sh;
8227         struct mlx5_flow_tbl_data_entry *tbl_data =
8228                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8229
8230         if (!tbl)
8231                 return 0;
8232         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
8233                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
8234
8235                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8236                 tbl->obj = NULL;
8237                 /* remove the entry from the hash list and free memory. */
8238                 mlx5_hlist_remove(sh->flow_tbls, pos);
8239                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
8240                                 tbl_data->idx);
8241                 return 0;
8242         }
8243         return 1;
8244 }
8245
8246 /**
8247  * Register the flow matcher.
8248  *
8249  * @param[in, out] dev
8250  *   Pointer to rte_eth_dev structure.
8251  * @param[in, out] matcher
8252  *   Pointer to flow matcher.
8253  * @param[in, out] key
8254  *   Pointer to flow table key.
8255  * @parm[in, out] dev_flow
8256  *   Pointer to the dev_flow.
8257  * @param[out] error
8258  *   pointer to error structure.
8259  *
8260  * @return
8261  *   0 on success otherwise -errno and errno is set.
8262  */
8263 static int
8264 flow_dv_matcher_register(struct rte_eth_dev *dev,
8265                          struct mlx5_flow_dv_matcher *matcher,
8266                          union mlx5_flow_tbl_key *key,
8267                          struct mlx5_flow *dev_flow,
8268                          struct rte_flow_error *error)
8269 {
8270         struct mlx5_priv *priv = dev->data->dev_private;
8271         struct mlx5_dev_ctx_shared *sh = priv->sh;
8272         struct mlx5_flow_dv_matcher *cache_matcher;
8273         struct mlx5dv_flow_matcher_attr dv_attr = {
8274                 .type = IBV_FLOW_ATTR_NORMAL,
8275                 .match_mask = (void *)&matcher->mask,
8276         };
8277         struct mlx5_flow_tbl_resource *tbl;
8278         struct mlx5_flow_tbl_data_entry *tbl_data;
8279         int ret;
8280
8281         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
8282                                        key->domain, error);
8283         if (!tbl)
8284                 return -rte_errno;      /* No need to refill the error info */
8285         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8286         /* Lookup from cache. */
8287         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
8288                 if (matcher->crc == cache_matcher->crc &&
8289                     matcher->priority == cache_matcher->priority &&
8290                     !memcmp((const void *)matcher->mask.buf,
8291                             (const void *)cache_matcher->mask.buf,
8292                             cache_matcher->mask.size)) {
8293                         DRV_LOG(DEBUG,
8294                                 "%s group %u priority %hd use %s "
8295                                 "matcher %p: refcnt %d++",
8296                                 key->domain ? "FDB" : "NIC", key->table_id,
8297                                 cache_matcher->priority,
8298                                 key->direction ? "tx" : "rx",
8299                                 (void *)cache_matcher,
8300                                 rte_atomic32_read(&cache_matcher->refcnt));
8301                         rte_atomic32_inc(&cache_matcher->refcnt);
8302                         dev_flow->handle->dvh.matcher = cache_matcher;
8303                         /* old matcher should not make the table ref++. */
8304                         flow_dv_tbl_resource_release(dev, tbl);
8305                         return 0;
8306                 }
8307         }
8308         /* Register new matcher. */
8309         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
8310                                     SOCKET_ID_ANY);
8311         if (!cache_matcher) {
8312                 flow_dv_tbl_resource_release(dev, tbl);
8313                 return rte_flow_error_set(error, ENOMEM,
8314                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8315                                           "cannot allocate matcher memory");
8316         }
8317         *cache_matcher = *matcher;
8318         dv_attr.match_criteria_enable =
8319                 flow_dv_matcher_enable(cache_matcher->mask.buf);
8320         dv_attr.priority = matcher->priority;
8321         if (key->direction)
8322                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
8323         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
8324                                                &cache_matcher->matcher_object);
8325         if (ret) {
8326                 mlx5_free(cache_matcher);
8327 #ifdef HAVE_MLX5DV_DR
8328                 flow_dv_tbl_resource_release(dev, tbl);
8329 #endif
8330                 return rte_flow_error_set(error, ENOMEM,
8331                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8332                                           NULL, "cannot create matcher");
8333         }
8334         /* Save the table information */
8335         cache_matcher->tbl = tbl;
8336         rte_atomic32_init(&cache_matcher->refcnt);
8337         /* only matcher ref++, table ref++ already done above in get API. */
8338         rte_atomic32_inc(&cache_matcher->refcnt);
8339         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
8340         dev_flow->handle->dvh.matcher = cache_matcher;
8341         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
8342                 key->domain ? "FDB" : "NIC", key->table_id,
8343                 cache_matcher->priority,
8344                 key->direction ? "tx" : "rx", (void *)cache_matcher,
8345                 rte_atomic32_read(&cache_matcher->refcnt));
8346         return 0;
8347 }
8348
8349 /**
8350  * Find existing tag resource or create and register a new one.
8351  *
8352  * @param dev[in, out]
8353  *   Pointer to rte_eth_dev structure.
8354  * @param[in, out] tag_be24
8355  *   Tag value in big endian then R-shift 8.
8356  * @parm[in, out] dev_flow
8357  *   Pointer to the dev_flow.
8358  * @param[out] error
8359  *   pointer to error structure.
8360  *
8361  * @return
8362  *   0 on success otherwise -errno and errno is set.
8363  */
8364 static int
8365 flow_dv_tag_resource_register
8366                         (struct rte_eth_dev *dev,
8367                          uint32_t tag_be24,
8368                          struct mlx5_flow *dev_flow,
8369                          struct rte_flow_error *error)
8370 {
8371         struct mlx5_priv *priv = dev->data->dev_private;
8372         struct mlx5_dev_ctx_shared *sh = priv->sh;
8373         struct mlx5_flow_dv_tag_resource *cache_resource;
8374         struct mlx5_hlist_entry *entry;
8375         int ret;
8376
8377         /* Lookup a matching resource from cache. */
8378         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
8379         if (entry) {
8380                 cache_resource = container_of
8381                         (entry, struct mlx5_flow_dv_tag_resource, entry);
8382                 rte_atomic32_inc(&cache_resource->refcnt);
8383                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
8384                 dev_flow->dv.tag_resource = cache_resource;
8385                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
8386                         (void *)cache_resource,
8387                         rte_atomic32_read(&cache_resource->refcnt));
8388                 return 0;
8389         }
8390         /* Register new resource. */
8391         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
8392                                        &dev_flow->handle->dvh.rix_tag);
8393         if (!cache_resource)
8394                 return rte_flow_error_set(error, ENOMEM,
8395                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8396                                           "cannot allocate resource memory");
8397         cache_resource->entry.key = (uint64_t)tag_be24;
8398         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
8399                                                   &cache_resource->action);
8400         if (ret) {
8401                 mlx5_free(cache_resource);
8402                 return rte_flow_error_set(error, ENOMEM,
8403                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8404                                           NULL, "cannot create action");
8405         }
8406         rte_atomic32_init(&cache_resource->refcnt);
8407         rte_atomic32_inc(&cache_resource->refcnt);
8408         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
8409                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
8410                 mlx5_free(cache_resource);
8411                 return rte_flow_error_set(error, EEXIST,
8412                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8413                                           NULL, "cannot insert tag");
8414         }
8415         dev_flow->dv.tag_resource = cache_resource;
8416         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
8417                 (void *)cache_resource,
8418                 rte_atomic32_read(&cache_resource->refcnt));
8419         return 0;
8420 }
8421
8422 /**
8423  * Release the tag.
8424  *
8425  * @param dev
8426  *   Pointer to Ethernet device.
8427  * @param tag_idx
8428  *   Tag index.
8429  *
8430  * @return
8431  *   1 while a reference on it exists, 0 when freed.
8432  */
8433 static int
8434 flow_dv_tag_release(struct rte_eth_dev *dev,
8435                     uint32_t tag_idx)
8436 {
8437         struct mlx5_priv *priv = dev->data->dev_private;
8438         struct mlx5_dev_ctx_shared *sh = priv->sh;
8439         struct mlx5_flow_dv_tag_resource *tag;
8440
8441         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8442         if (!tag)
8443                 return 0;
8444         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
8445                 dev->data->port_id, (void *)tag,
8446                 rte_atomic32_read(&tag->refcnt));
8447         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
8448                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
8449                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
8450                 DRV_LOG(DEBUG, "port %u tag %p: removed",
8451                         dev->data->port_id, (void *)tag);
8452                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8453                 return 0;
8454         }
8455         return 1;
8456 }
8457
8458 /**
8459  * Translate port ID action to vport.
8460  *
8461  * @param[in] dev
8462  *   Pointer to rte_eth_dev structure.
8463  * @param[in] action
8464  *   Pointer to the port ID action.
8465  * @param[out] dst_port_id
8466  *   The target port ID.
8467  * @param[out] error
8468  *   Pointer to the error structure.
8469  *
8470  * @return
8471  *   0 on success, a negative errno value otherwise and rte_errno is set.
8472  */
8473 static int
8474 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
8475                                  const struct rte_flow_action *action,
8476                                  uint32_t *dst_port_id,
8477                                  struct rte_flow_error *error)
8478 {
8479         uint32_t port;
8480         struct mlx5_priv *priv;
8481         const struct rte_flow_action_port_id *conf =
8482                         (const struct rte_flow_action_port_id *)action->conf;
8483
8484         port = conf->original ? dev->data->port_id : conf->id;
8485         priv = mlx5_port_to_eswitch_info(port, false);
8486         if (!priv)
8487                 return rte_flow_error_set(error, -rte_errno,
8488                                           RTE_FLOW_ERROR_TYPE_ACTION,
8489                                           NULL,
8490                                           "No eswitch info was found for port");
8491 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
8492         /*
8493          * This parameter is transferred to
8494          * mlx5dv_dr_action_create_dest_ib_port().
8495          */
8496         *dst_port_id = priv->dev_port;
8497 #else
8498         /*
8499          * Legacy mode, no LAG configurations is supported.
8500          * This parameter is transferred to
8501          * mlx5dv_dr_action_create_dest_vport().
8502          */
8503         *dst_port_id = priv->vport_id;
8504 #endif
8505         return 0;
8506 }
8507
8508 /**
8509  * Create a counter with aging configuration.
8510  *
8511  * @param[in] dev
8512  *   Pointer to rte_eth_dev structure.
8513  * @param[out] count
8514  *   Pointer to the counter action configuration.
8515  * @param[in] age
8516  *   Pointer to the aging action configuration.
8517  *
8518  * @return
8519  *   Index to flow counter on success, 0 otherwise.
8520  */
8521 static uint32_t
8522 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
8523                                 struct mlx5_flow *dev_flow,
8524                                 const struct rte_flow_action_count *count,
8525                                 const struct rte_flow_action_age *age)
8526 {
8527         uint32_t counter;
8528         struct mlx5_age_param *age_param;
8529
8530         counter = flow_dv_counter_alloc(dev,
8531                                 count ? count->shared : 0,
8532                                 count ? count->id : 0,
8533                                 dev_flow->dv.group, !!age);
8534         if (!counter || age == NULL)
8535                 return counter;
8536         age_param  = flow_dv_counter_idx_get_age(dev, counter);
8537         age_param->context = age->context ? age->context :
8538                 (void *)(uintptr_t)(dev_flow->flow_idx);
8539         age_param->timeout = age->timeout;
8540         age_param->port_id = dev->data->port_id;
8541         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
8542         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
8543         return counter;
8544 }
8545 /**
8546  * Add Tx queue matcher
8547  *
8548  * @param[in] dev
8549  *   Pointer to the dev struct.
8550  * @param[in, out] matcher
8551  *   Flow matcher.
8552  * @param[in, out] key
8553  *   Flow matcher value.
8554  * @param[in] item
8555  *   Flow pattern to translate.
8556  * @param[in] inner
8557  *   Item is inner pattern.
8558  */
8559 static void
8560 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
8561                                 void *matcher, void *key,
8562                                 const struct rte_flow_item *item)
8563 {
8564         const struct mlx5_rte_flow_item_tx_queue *queue_m;
8565         const struct mlx5_rte_flow_item_tx_queue *queue_v;
8566         void *misc_m =
8567                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8568         void *misc_v =
8569                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8570         struct mlx5_txq_ctrl *txq;
8571         uint32_t queue;
8572
8573
8574         queue_m = (const void *)item->mask;
8575         if (!queue_m)
8576                 return;
8577         queue_v = (const void *)item->spec;
8578         if (!queue_v)
8579                 return;
8580         txq = mlx5_txq_get(dev, queue_v->queue);
8581         if (!txq)
8582                 return;
8583         queue = txq->obj->sq->id;
8584         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
8585         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
8586                  queue & queue_m->queue);
8587         mlx5_txq_release(dev, queue_v->queue);
8588 }
8589
8590 /**
8591  * Set the hash fields according to the @p flow information.
8592  *
8593  * @param[in] dev_flow
8594  *   Pointer to the mlx5_flow.
8595  * @param[in] rss_desc
8596  *   Pointer to the mlx5_flow_rss_desc.
8597  */
8598 static void
8599 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
8600                        struct mlx5_flow_rss_desc *rss_desc)
8601 {
8602         uint64_t items = dev_flow->handle->layers;
8603         int rss_inner = 0;
8604         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
8605
8606         dev_flow->hash_fields = 0;
8607 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
8608         if (rss_desc->level >= 2) {
8609                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
8610                 rss_inner = 1;
8611         }
8612 #endif
8613         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
8614             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
8615                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
8616                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8617                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
8618                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8619                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
8620                         else
8621                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
8622                 }
8623         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
8624                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
8625                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
8626                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8627                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
8628                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8629                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
8630                         else
8631                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
8632                 }
8633         }
8634         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
8635             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
8636                 if (rss_types & ETH_RSS_UDP) {
8637                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8638                                 dev_flow->hash_fields |=
8639                                                 IBV_RX_HASH_SRC_PORT_UDP;
8640                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8641                                 dev_flow->hash_fields |=
8642                                                 IBV_RX_HASH_DST_PORT_UDP;
8643                         else
8644                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
8645                 }
8646         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
8647                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
8648                 if (rss_types & ETH_RSS_TCP) {
8649                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8650                                 dev_flow->hash_fields |=
8651                                                 IBV_RX_HASH_SRC_PORT_TCP;
8652                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8653                                 dev_flow->hash_fields |=
8654                                                 IBV_RX_HASH_DST_PORT_TCP;
8655                         else
8656                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
8657                 }
8658         }
8659 }
8660
8661 /**
8662  * Create an Rx Hash queue.
8663  *
8664  * @param dev
8665  *   Pointer to Ethernet device.
8666  * @param[in] dev_flow
8667  *   Pointer to the mlx5_flow.
8668  * @param[in] rss_desc
8669  *   Pointer to the mlx5_flow_rss_desc.
8670  * @param[out] hrxq_idx
8671  *   Hash Rx queue index.
8672  *
8673  * @return
8674  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
8675  */
8676 static struct mlx5_hrxq *
8677 flow_dv_handle_rx_queue(struct rte_eth_dev *dev,
8678                         struct mlx5_flow *dev_flow,
8679                         struct mlx5_flow_rss_desc *rss_desc,
8680                         uint32_t *hrxq_idx)
8681 {
8682         struct mlx5_priv *priv = dev->data->dev_private;
8683         struct mlx5_flow_handle *dh = dev_flow->handle;
8684         struct mlx5_hrxq *hrxq;
8685
8686         MLX5_ASSERT(rss_desc->queue_num);
8687         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8688                                   MLX5_RSS_HASH_KEY_LEN,
8689                                   dev_flow->hash_fields,
8690                                   rss_desc->queue,
8691                                   rss_desc->queue_num);
8692         if (!*hrxq_idx) {
8693                 *hrxq_idx = mlx5_hrxq_new
8694                                 (dev, rss_desc->key,
8695                                  MLX5_RSS_HASH_KEY_LEN,
8696                                  dev_flow->hash_fields,
8697                                  rss_desc->queue,
8698                                  rss_desc->queue_num,
8699                                  !!(dh->layers &
8700                                  MLX5_FLOW_LAYER_TUNNEL));
8701                 if (!*hrxq_idx)
8702                         return NULL;
8703         }
8704         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8705                               *hrxq_idx);
8706         return hrxq;
8707 }
8708
8709 /**
8710  * Find existing sample resource or create and register a new one.
8711  *
8712  * @param[in, out] dev
8713  *   Pointer to rte_eth_dev structure.
8714  * @param[in] attr
8715  *   Attributes of flow that includes this item.
8716  * @param[in] resource
8717  *   Pointer to sample resource.
8718  * @parm[in, out] dev_flow
8719  *   Pointer to the dev_flow.
8720  * @param[in, out] sample_dv_actions
8721  *   Pointer to sample actions list.
8722  * @param[out] error
8723  *   pointer to error structure.
8724  *
8725  * @return
8726  *   0 on success otherwise -errno and errno is set.
8727  */
8728 static int
8729 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
8730                          const struct rte_flow_attr *attr,
8731                          struct mlx5_flow_dv_sample_resource *resource,
8732                          struct mlx5_flow *dev_flow,
8733                          void **sample_dv_actions,
8734                          struct rte_flow_error *error)
8735 {
8736         struct mlx5_flow_dv_sample_resource *cache_resource;
8737         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
8738         struct mlx5_priv *priv = dev->data->dev_private;
8739         struct mlx5_dev_ctx_shared *sh = priv->sh;
8740         struct mlx5_flow_tbl_resource *tbl;
8741         uint32_t idx = 0;
8742         const uint32_t next_ft_step = 1;
8743         uint32_t next_ft_id = resource->ft_id + next_ft_step;
8744
8745         /* Lookup a matching resource from cache. */
8746         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
8747                       idx, cache_resource, next) {
8748                 if (resource->ratio == cache_resource->ratio &&
8749                     resource->ft_type == cache_resource->ft_type &&
8750                     resource->ft_id == cache_resource->ft_id &&
8751                     resource->set_action == cache_resource->set_action &&
8752                     !memcmp((void *)&resource->sample_act,
8753                             (void *)&cache_resource->sample_act,
8754                             sizeof(struct mlx5_flow_sub_actions_list))) {
8755                         DRV_LOG(DEBUG, "sample resource %p: refcnt %d++",
8756                                 (void *)cache_resource,
8757                                 __atomic_load_n(&cache_resource->refcnt,
8758                                                 __ATOMIC_RELAXED));
8759                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8760                                            __ATOMIC_RELAXED);
8761                         dev_flow->handle->dvh.rix_sample = idx;
8762                         dev_flow->dv.sample_res = cache_resource;
8763                         return 0;
8764                 }
8765         }
8766         /* Register new sample resource. */
8767         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE],
8768                                        &dev_flow->handle->dvh.rix_sample);
8769         if (!cache_resource)
8770                 return rte_flow_error_set(error, ENOMEM,
8771                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8772                                           NULL,
8773                                           "cannot allocate resource memory");
8774         *cache_resource = *resource;
8775         /* Create normal path table level */
8776         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
8777                                         attr->egress, attr->transfer, error);
8778         if (!tbl) {
8779                 rte_flow_error_set(error, ENOMEM,
8780                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8781                                           NULL,
8782                                           "fail to create normal path table "
8783                                           "for sample");
8784                 goto error;
8785         }
8786         cache_resource->normal_path_tbl = tbl;
8787         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8788                 cache_resource->default_miss =
8789                                 mlx5_glue->dr_create_flow_action_default_miss();
8790                 if (!cache_resource->default_miss) {
8791                         rte_flow_error_set(error, ENOMEM,
8792                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8793                                                 NULL,
8794                                                 "cannot create default miss "
8795                                                 "action");
8796                         goto error;
8797                 }
8798                 sample_dv_actions[resource->sample_act.actions_num++] =
8799                                                 cache_resource->default_miss;
8800         }
8801         /* Create a DR sample action */
8802         sampler_attr.sample_ratio = cache_resource->ratio;
8803         sampler_attr.default_next_table = tbl->obj;
8804         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
8805         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
8806                                                         &sample_dv_actions[0];
8807         sampler_attr.action = cache_resource->set_action;
8808         cache_resource->verbs_action =
8809                 mlx5_glue->dr_create_flow_action_sampler(&sampler_attr);
8810         if (!cache_resource->verbs_action) {
8811                 rte_flow_error_set(error, ENOMEM,
8812                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8813                                         NULL, "cannot create sample action");
8814                 goto error;
8815         }
8816         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8817         ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list,
8818                      dev_flow->handle->dvh.rix_sample, cache_resource,
8819                      next);
8820         dev_flow->dv.sample_res = cache_resource;
8821         DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++",
8822                 (void *)cache_resource,
8823                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8824         return 0;
8825 error:
8826         if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8827                 if (cache_resource->default_miss)
8828                         claim_zero(mlx5_glue->destroy_flow_action
8829                                 (cache_resource->default_miss));
8830         } else {
8831                 if (cache_resource->sample_idx.rix_hrxq &&
8832                     !mlx5_hrxq_release(dev,
8833                                 cache_resource->sample_idx.rix_hrxq))
8834                         cache_resource->sample_idx.rix_hrxq = 0;
8835                 if (cache_resource->sample_idx.rix_tag &&
8836                     !flow_dv_tag_release(dev,
8837                                 cache_resource->sample_idx.rix_tag))
8838                         cache_resource->sample_idx.rix_tag = 0;
8839                 if (cache_resource->sample_idx.cnt) {
8840                         flow_dv_counter_release(dev,
8841                                 cache_resource->sample_idx.cnt);
8842                         cache_resource->sample_idx.cnt = 0;
8843                 }
8844         }
8845         if (cache_resource->normal_path_tbl)
8846                 flow_dv_tbl_resource_release(dev,
8847                                 cache_resource->normal_path_tbl);
8848         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE],
8849                                 dev_flow->handle->dvh.rix_sample);
8850         dev_flow->handle->dvh.rix_sample = 0;
8851         return -rte_errno;
8852 }
8853
8854 /**
8855  * Find existing destination array resource or create and register a new one.
8856  *
8857  * @param[in, out] dev
8858  *   Pointer to rte_eth_dev structure.
8859  * @param[in] attr
8860  *   Attributes of flow that includes this item.
8861  * @param[in] resource
8862  *   Pointer to destination array resource.
8863  * @parm[in, out] dev_flow
8864  *   Pointer to the dev_flow.
8865  * @param[out] error
8866  *   pointer to error structure.
8867  *
8868  * @return
8869  *   0 on success otherwise -errno and errno is set.
8870  */
8871 static int
8872 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
8873                          const struct rte_flow_attr *attr,
8874                          struct mlx5_flow_dv_dest_array_resource *resource,
8875                          struct mlx5_flow *dev_flow,
8876                          struct rte_flow_error *error)
8877 {
8878         struct mlx5_flow_dv_dest_array_resource *cache_resource;
8879         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
8880         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
8881         struct mlx5_priv *priv = dev->data->dev_private;
8882         struct mlx5_dev_ctx_shared *sh = priv->sh;
8883         struct mlx5_flow_sub_actions_list *sample_act;
8884         struct mlx5dv_dr_domain *domain;
8885         uint32_t idx = 0;
8886
8887         /* Lookup a matching resource from cache. */
8888         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8889                       sh->dest_array_list,
8890                       idx, cache_resource, next) {
8891                 if (resource->num_of_dest == cache_resource->num_of_dest &&
8892                     resource->ft_type == cache_resource->ft_type &&
8893                     !memcmp((void *)cache_resource->sample_act,
8894                             (void *)resource->sample_act,
8895                            (resource->num_of_dest *
8896                            sizeof(struct mlx5_flow_sub_actions_list)))) {
8897                         DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++",
8898                                 (void *)cache_resource,
8899                                 __atomic_load_n(&cache_resource->refcnt,
8900                                                 __ATOMIC_RELAXED));
8901                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8902                                            __ATOMIC_RELAXED);
8903                         dev_flow->handle->dvh.rix_dest_array = idx;
8904                         dev_flow->dv.dest_array_res = cache_resource;
8905                         return 0;
8906                 }
8907         }
8908         /* Register new destination array resource. */
8909         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8910                                        &dev_flow->handle->dvh.rix_dest_array);
8911         if (!cache_resource)
8912                 return rte_flow_error_set(error, ENOMEM,
8913                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8914                                           NULL,
8915                                           "cannot allocate resource memory");
8916         *cache_resource = *resource;
8917         if (attr->transfer)
8918                 domain = sh->fdb_domain;
8919         else if (attr->ingress)
8920                 domain = sh->rx_domain;
8921         else
8922                 domain = sh->tx_domain;
8923         for (idx = 0; idx < resource->num_of_dest; idx++) {
8924                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
8925                                  mlx5_malloc(MLX5_MEM_ZERO,
8926                                  sizeof(struct mlx5dv_dr_action_dest_attr),
8927                                  0, SOCKET_ID_ANY);
8928                 if (!dest_attr[idx]) {
8929                         rte_flow_error_set(error, ENOMEM,
8930                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8931                                            NULL,
8932                                            "cannot allocate resource memory");
8933                         goto error;
8934                 }
8935                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
8936                 sample_act = &resource->sample_act[idx];
8937                 if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) {
8938                         dest_attr[idx]->dest = sample_act->dr_queue_action;
8939                 } else if (sample_act->action_flags ==
8940                           (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) {
8941                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
8942                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
8943                         dest_attr[idx]->dest_reformat->reformat =
8944                                         sample_act->dr_encap_action;
8945                         dest_attr[idx]->dest_reformat->dest =
8946                                         sample_act->dr_port_id_action;
8947                 } else if (sample_act->action_flags ==
8948                            MLX5_FLOW_ACTION_PORT_ID) {
8949                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
8950                 }
8951         }
8952         /* create a dest array actioin */
8953         cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array
8954                                                 (domain,
8955                                                  cache_resource->num_of_dest,
8956                                                  dest_attr);
8957         if (!cache_resource->action) {
8958                 rte_flow_error_set(error, ENOMEM,
8959                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8960                                    NULL,
8961                                    "cannot create destination array action");
8962                 goto error;
8963         }
8964         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8965         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8966                      &sh->dest_array_list,
8967                      dev_flow->handle->dvh.rix_dest_array, cache_resource,
8968                      next);
8969         dev_flow->dv.dest_array_res = cache_resource;
8970         DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++",
8971                 (void *)cache_resource,
8972                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8973         for (idx = 0; idx < resource->num_of_dest; idx++)
8974                 mlx5_free(dest_attr[idx]);
8975         return 0;
8976 error:
8977         for (idx = 0; idx < resource->num_of_dest; idx++) {
8978                 struct mlx5_flow_sub_actions_idx *act_res =
8979                                         &cache_resource->sample_idx[idx];
8980                 if (act_res->rix_hrxq &&
8981                     !mlx5_hrxq_release(dev,
8982                                 act_res->rix_hrxq))
8983                         act_res->rix_hrxq = 0;
8984                 if (act_res->rix_encap_decap &&
8985                         !flow_dv_encap_decap_resource_release(dev,
8986                                 act_res->rix_encap_decap))
8987                         act_res->rix_encap_decap = 0;
8988                 if (act_res->rix_port_id_action &&
8989                         !flow_dv_port_id_action_resource_release(dev,
8990                                 act_res->rix_port_id_action))
8991                         act_res->rix_port_id_action = 0;
8992                 if (dest_attr[idx])
8993                         mlx5_free(dest_attr[idx]);
8994         }
8995
8996         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8997                                 dev_flow->handle->dvh.rix_dest_array);
8998         dev_flow->handle->dvh.rix_dest_array = 0;
8999         return -rte_errno;
9000 }
9001
9002 /**
9003  * Convert Sample action to DV specification.
9004  *
9005  * @param[in] dev
9006  *   Pointer to rte_eth_dev structure.
9007  * @param[in] action
9008  *   Pointer to action structure.
9009  * @param[in, out] dev_flow
9010  *   Pointer to the mlx5_flow.
9011  * @param[in] attr
9012  *   Pointer to the flow attributes.
9013  * @param[in, out] num_of_dest
9014  *   Pointer to the num of destination.
9015  * @param[in, out] sample_actions
9016  *   Pointer to sample actions list.
9017  * @param[in, out] res
9018  *   Pointer to sample resource.
9019  * @param[out] error
9020  *   Pointer to the error structure.
9021  *
9022  * @return
9023  *   0 on success, a negative errno value otherwise and rte_errno is set.
9024  */
9025 static int
9026 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
9027                                 const struct rte_flow_action *action,
9028                                 struct mlx5_flow *dev_flow,
9029                                 const struct rte_flow_attr *attr,
9030                                 uint32_t *num_of_dest,
9031                                 void **sample_actions,
9032                                 struct mlx5_flow_dv_sample_resource *res,
9033                                 struct rte_flow_error *error)
9034 {
9035         struct mlx5_priv *priv = dev->data->dev_private;
9036         const struct rte_flow_action_sample *sample_action;
9037         const struct rte_flow_action *sub_actions;
9038         const struct rte_flow_action_queue *queue;
9039         struct mlx5_flow_sub_actions_list *sample_act;
9040         struct mlx5_flow_sub_actions_idx *sample_idx;
9041         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9042                                               priv->rss_desc)
9043                                               [!!priv->flow_nested_idx];
9044         uint64_t action_flags = 0;
9045
9046         sample_act = &res->sample_act;
9047         sample_idx = &res->sample_idx;
9048         sample_action = (const struct rte_flow_action_sample *)action->conf;
9049         res->ratio = sample_action->ratio;
9050         sub_actions = sample_action->actions;
9051         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
9052                 int type = sub_actions->type;
9053                 uint32_t pre_rix = 0;
9054                 void *pre_r;
9055                 switch (type) {
9056                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9057                 {
9058                         struct mlx5_hrxq *hrxq;
9059                         uint32_t hrxq_idx;
9060
9061                         queue = sub_actions->conf;
9062                         rss_desc->queue_num = 1;
9063                         rss_desc->queue[0] = queue->index;
9064                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9065                                         rss_desc, &hrxq_idx);
9066                         if (!hrxq)
9067                                 return rte_flow_error_set
9068                                         (error, rte_errno,
9069                                          RTE_FLOW_ERROR_TYPE_ACTION,
9070                                          NULL,
9071                                          "cannot create fate queue");
9072                         sample_act->dr_queue_action = hrxq->action;
9073                         sample_idx->rix_hrxq = hrxq_idx;
9074                         sample_actions[sample_act->actions_num++] =
9075                                                 hrxq->action;
9076                         (*num_of_dest)++;
9077                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9078                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9079                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9080                         dev_flow->handle->fate_action =
9081                                         MLX5_FLOW_FATE_QUEUE;
9082                         break;
9083                 }
9084                 case RTE_FLOW_ACTION_TYPE_MARK:
9085                 {
9086                         uint32_t tag_be = mlx5_flow_mark_set
9087                                 (((const struct rte_flow_action_mark *)
9088                                 (sub_actions->conf))->id);
9089
9090                         dev_flow->handle->mark = 1;
9091                         pre_rix = dev_flow->handle->dvh.rix_tag;
9092                         /* Save the mark resource before sample */
9093                         pre_r = dev_flow->dv.tag_resource;
9094                         if (flow_dv_tag_resource_register(dev, tag_be,
9095                                                   dev_flow, error))
9096                                 return -rte_errno;
9097                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9098                         sample_act->dr_tag_action =
9099                                 dev_flow->dv.tag_resource->action;
9100                         sample_idx->rix_tag =
9101                                 dev_flow->handle->dvh.rix_tag;
9102                         sample_actions[sample_act->actions_num++] =
9103                                                 sample_act->dr_tag_action;
9104                         /* Recover the mark resource after sample */
9105                         dev_flow->dv.tag_resource = pre_r;
9106                         dev_flow->handle->dvh.rix_tag = pre_rix;
9107                         action_flags |= MLX5_FLOW_ACTION_MARK;
9108                         break;
9109                 }
9110                 case RTE_FLOW_ACTION_TYPE_COUNT:
9111                 {
9112                         uint32_t counter;
9113
9114                         counter = flow_dv_translate_create_counter(dev,
9115                                         dev_flow, sub_actions->conf, 0);
9116                         if (!counter)
9117                                 return rte_flow_error_set
9118                                                 (error, rte_errno,
9119                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9120                                                  NULL,
9121                                                  "cannot create counter"
9122                                                  " object.");
9123                         sample_idx->cnt = counter;
9124                         sample_act->dr_cnt_action =
9125                                   (flow_dv_counter_get_by_idx(dev,
9126                                   counter, NULL))->action;
9127                         sample_actions[sample_act->actions_num++] =
9128                                                 sample_act->dr_cnt_action;
9129                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9130                         break;
9131                 }
9132                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9133                 {
9134                         struct mlx5_flow_dv_port_id_action_resource
9135                                         port_id_resource;
9136                         uint32_t port_id = 0;
9137
9138                         memset(&port_id_resource, 0, sizeof(port_id_resource));
9139                         /* Save the port id resource before sample */
9140                         pre_rix = dev_flow->handle->rix_port_id_action;
9141                         pre_r = dev_flow->dv.port_id_action;
9142                         if (flow_dv_translate_action_port_id(dev, sub_actions,
9143                                                              &port_id, error))
9144                                 return -rte_errno;
9145                         port_id_resource.port_id = port_id;
9146                         if (flow_dv_port_id_action_resource_register
9147                             (dev, &port_id_resource, dev_flow, error))
9148                                 return -rte_errno;
9149                         sample_act->dr_port_id_action =
9150                                 dev_flow->dv.port_id_action->action;
9151                         sample_idx->rix_port_id_action =
9152                                 dev_flow->handle->rix_port_id_action;
9153                         sample_actions[sample_act->actions_num++] =
9154                                                 sample_act->dr_port_id_action;
9155                         /* Recover the port id resource after sample */
9156                         dev_flow->dv.port_id_action = pre_r;
9157                         dev_flow->handle->rix_port_id_action = pre_rix;
9158                         (*num_of_dest)++;
9159                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9160                         break;
9161                 }
9162                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9163                         /* Save the encap resource before sample */
9164                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
9165                         pre_r = dev_flow->dv.encap_decap;
9166                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
9167                                                            dev_flow,
9168                                                            attr->transfer,
9169                                                            error))
9170                                 return -rte_errno;
9171                         sample_act->dr_encap_action =
9172                                 dev_flow->dv.encap_decap->action;
9173                         sample_idx->rix_encap_decap =
9174                                 dev_flow->handle->dvh.rix_encap_decap;
9175                         sample_actions[sample_act->actions_num++] =
9176                                                 sample_act->dr_encap_action;
9177                         /* Recover the encap resource after sample */
9178                         dev_flow->dv.encap_decap = pre_r;
9179                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
9180                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9181                         break;
9182                 default:
9183                         return rte_flow_error_set(error, EINVAL,
9184                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9185                                 NULL,
9186                                 "Not support for sampler action");
9187                 }
9188         }
9189         sample_act->action_flags = action_flags;
9190         res->ft_id = dev_flow->dv.group;
9191         if (attr->transfer) {
9192                 union {
9193                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
9194                         uint64_t set_action;
9195                 } action_ctx = { .set_action = 0 };
9196
9197                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9198                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
9199                          MLX5_MODIFICATION_TYPE_SET);
9200                 MLX5_SET(set_action_in, action_ctx.action_in, field,
9201                          MLX5_MODI_META_REG_C_0);
9202                 MLX5_SET(set_action_in, action_ctx.action_in, data,
9203                          priv->vport_meta_tag);
9204                 res->set_action = action_ctx.set_action;
9205         } else if (attr->ingress) {
9206                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9207         }
9208         return 0;
9209 }
9210
9211 /**
9212  * Convert Sample action to DV specification.
9213  *
9214  * @param[in] dev
9215  *   Pointer to rte_eth_dev structure.
9216  * @param[in, out] dev_flow
9217  *   Pointer to the mlx5_flow.
9218  * @param[in] attr
9219  *   Pointer to the flow attributes.
9220  * @param[in] num_of_dest
9221  *   The num of destination.
9222  * @param[in, out] res
9223  *   Pointer to sample resource.
9224  * @param[in, out] mdest_res
9225  *   Pointer to destination array resource.
9226  * @param[in] sample_actions
9227  *   Pointer to sample path actions list.
9228  * @param[in] action_flags
9229  *   Holds the actions detected until now.
9230  * @param[out] error
9231  *   Pointer to the error structure.
9232  *
9233  * @return
9234  *   0 on success, a negative errno value otherwise and rte_errno is set.
9235  */
9236 static int
9237 flow_dv_create_action_sample(struct rte_eth_dev *dev,
9238                              struct mlx5_flow *dev_flow,
9239                              const struct rte_flow_attr *attr,
9240                              uint32_t num_of_dest,
9241                              struct mlx5_flow_dv_sample_resource *res,
9242                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
9243                              void **sample_actions,
9244                              uint64_t action_flags,
9245                              struct rte_flow_error *error)
9246 {
9247         struct mlx5_priv *priv = dev->data->dev_private;
9248         /* update normal path action resource into last index of array */
9249         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
9250         struct mlx5_flow_sub_actions_list *sample_act =
9251                                         &mdest_res->sample_act[dest_index];
9252         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9253                                               priv->rss_desc)
9254                                               [!!priv->flow_nested_idx];
9255         uint32_t normal_idx = 0;
9256         struct mlx5_hrxq *hrxq;
9257         uint32_t hrxq_idx;
9258
9259         if (num_of_dest > 1) {
9260                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
9261                         /* Handle QP action for mirroring */
9262                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9263                                                        rss_desc, &hrxq_idx);
9264                         if (!hrxq)
9265                                 return rte_flow_error_set
9266                                      (error, rte_errno,
9267                                       RTE_FLOW_ERROR_TYPE_ACTION,
9268                                       NULL,
9269                                       "cannot create rx queue");
9270                         normal_idx++;
9271                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
9272                         sample_act->dr_queue_action = hrxq->action;
9273                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9274                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9275                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9276                 }
9277                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
9278                         normal_idx++;
9279                         mdest_res->sample_idx[dest_index].rix_encap_decap =
9280                                 dev_flow->handle->dvh.rix_encap_decap;
9281                         sample_act->dr_encap_action =
9282                                 dev_flow->dv.encap_decap->action;
9283                 }
9284                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
9285                         normal_idx++;
9286                         mdest_res->sample_idx[dest_index].rix_port_id_action =
9287                                 dev_flow->handle->rix_port_id_action;
9288                         sample_act->dr_port_id_action =
9289                                 dev_flow->dv.port_id_action->action;
9290                 }
9291                 sample_act->actions_num = normal_idx;
9292                 /* update sample action resource into first index of array */
9293                 mdest_res->ft_type = res->ft_type;
9294                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
9295                                 sizeof(struct mlx5_flow_sub_actions_idx));
9296                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
9297                                 sizeof(struct mlx5_flow_sub_actions_list));
9298                 mdest_res->num_of_dest = num_of_dest;
9299                 if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
9300                                                          dev_flow, error))
9301                         return rte_flow_error_set(error, EINVAL,
9302                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9303                                                   NULL, "can't create sample "
9304                                                   "action");
9305         } else {
9306                 if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
9307                                                      sample_actions, error))
9308                         return rte_flow_error_set(error, EINVAL,
9309                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9310                                                   NULL,
9311                                                   "can't create sample action");
9312         }
9313         return 0;
9314 }
9315
9316 /**
9317  * Fill the flow with DV spec, lock free
9318  * (mutex should be acquired by caller).
9319  *
9320  * @param[in] dev
9321  *   Pointer to rte_eth_dev structure.
9322  * @param[in, out] dev_flow
9323  *   Pointer to the sub flow.
9324  * @param[in] attr
9325  *   Pointer to the flow attributes.
9326  * @param[in] items
9327  *   Pointer to the list of items.
9328  * @param[in] actions
9329  *   Pointer to the list of actions.
9330  * @param[out] error
9331  *   Pointer to the error structure.
9332  *
9333  * @return
9334  *   0 on success, a negative errno value otherwise and rte_errno is set.
9335  */
9336 static int
9337 __flow_dv_translate(struct rte_eth_dev *dev,
9338                     struct mlx5_flow *dev_flow,
9339                     const struct rte_flow_attr *attr,
9340                     const struct rte_flow_item items[],
9341                     const struct rte_flow_action actions[],
9342                     struct rte_flow_error *error)
9343 {
9344         struct mlx5_priv *priv = dev->data->dev_private;
9345         struct mlx5_dev_config *dev_conf = &priv->config;
9346         struct rte_flow *flow = dev_flow->flow;
9347         struct mlx5_flow_handle *handle = dev_flow->handle;
9348         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9349                                               priv->rss_desc)
9350                                               [!!priv->flow_nested_idx];
9351         uint64_t item_flags = 0;
9352         uint64_t last_item = 0;
9353         uint64_t action_flags = 0;
9354         uint64_t priority = attr->priority;
9355         struct mlx5_flow_dv_matcher matcher = {
9356                 .mask = {
9357                         .size = sizeof(matcher.mask.buf) -
9358                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
9359                 },
9360         };
9361         int actions_n = 0;
9362         bool actions_end = false;
9363         union {
9364                 struct mlx5_flow_dv_modify_hdr_resource res;
9365                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
9366                             sizeof(struct mlx5_modification_cmd) *
9367                             (MLX5_MAX_MODIFY_NUM + 1)];
9368         } mhdr_dummy;
9369         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
9370         const struct rte_flow_action_count *count = NULL;
9371         const struct rte_flow_action_age *age = NULL;
9372         union flow_dv_attr flow_attr = { .attr = 0 };
9373         uint32_t tag_be;
9374         union mlx5_flow_tbl_key tbl_key;
9375         uint32_t modify_action_position = UINT32_MAX;
9376         void *match_mask = matcher.mask.buf;
9377         void *match_value = dev_flow->dv.value.buf;
9378         uint8_t next_protocol = 0xff;
9379         struct rte_vlan_hdr vlan = { 0 };
9380         struct mlx5_flow_dv_dest_array_resource mdest_res;
9381         struct mlx5_flow_dv_sample_resource sample_res;
9382         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9383         struct mlx5_flow_sub_actions_list *sample_act;
9384         uint32_t sample_act_pos = UINT32_MAX;
9385         uint32_t num_of_dest = 0;
9386         int tmp_actions_n = 0;
9387         uint32_t table;
9388         int ret = 0;
9389
9390         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
9391         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
9392         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
9393                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9394         /* update normal path action resource into last index of array */
9395         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
9396         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
9397                                        !!priv->fdb_def_rule, &table, error);
9398         if (ret)
9399                 return ret;
9400         dev_flow->dv.group = table;
9401         if (attr->transfer)
9402                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9403         if (priority == MLX5_FLOW_PRIO_RSVD)
9404                 priority = dev_conf->flow_prio - 1;
9405         /* number of actions must be set to 0 in case of dirty stack. */
9406         mhdr_res->actions_num = 0;
9407         for (; !actions_end ; actions++) {
9408                 const struct rte_flow_action_queue *queue;
9409                 const struct rte_flow_action_rss *rss;
9410                 const struct rte_flow_action *action = actions;
9411                 const uint8_t *rss_key;
9412                 const struct rte_flow_action_meter *mtr;
9413                 struct mlx5_flow_tbl_resource *tbl;
9414                 uint32_t port_id = 0;
9415                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
9416                 int action_type = actions->type;
9417                 const struct rte_flow_action *found_action = NULL;
9418                 struct mlx5_flow_meter *fm = NULL;
9419                 uint32_t jump_group = 0;
9420
9421                 if (!mlx5_flow_os_action_supported(action_type))
9422                         return rte_flow_error_set(error, ENOTSUP,
9423                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9424                                                   actions,
9425                                                   "action not supported");
9426                 switch (action_type) {
9427                 case RTE_FLOW_ACTION_TYPE_VOID:
9428                         break;
9429                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9430                         if (flow_dv_translate_action_port_id(dev, action,
9431                                                              &port_id, error))
9432                                 return -rte_errno;
9433                         port_id_resource.port_id = port_id;
9434                         MLX5_ASSERT(!handle->rix_port_id_action);
9435                         if (flow_dv_port_id_action_resource_register
9436                             (dev, &port_id_resource, dev_flow, error))
9437                                 return -rte_errno;
9438                         dev_flow->dv.actions[actions_n++] =
9439                                         dev_flow->dv.port_id_action->action;
9440                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9441                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
9442                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9443                         num_of_dest++;
9444                         break;
9445                 case RTE_FLOW_ACTION_TYPE_FLAG:
9446                         action_flags |= MLX5_FLOW_ACTION_FLAG;
9447                         dev_flow->handle->mark = 1;
9448                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9449                                 struct rte_flow_action_mark mark = {
9450                                         .id = MLX5_FLOW_MARK_DEFAULT,
9451                                 };
9452
9453                                 if (flow_dv_convert_action_mark(dev, &mark,
9454                                                                 mhdr_res,
9455                                                                 error))
9456                                         return -rte_errno;
9457                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9458                                 break;
9459                         }
9460                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
9461                         /*
9462                          * Only one FLAG or MARK is supported per device flow
9463                          * right now. So the pointer to the tag resource must be
9464                          * zero before the register process.
9465                          */
9466                         MLX5_ASSERT(!handle->dvh.rix_tag);
9467                         if (flow_dv_tag_resource_register(dev, tag_be,
9468                                                           dev_flow, error))
9469                                 return -rte_errno;
9470                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9471                         dev_flow->dv.actions[actions_n++] =
9472                                         dev_flow->dv.tag_resource->action;
9473                         break;
9474                 case RTE_FLOW_ACTION_TYPE_MARK:
9475                         action_flags |= MLX5_FLOW_ACTION_MARK;
9476                         dev_flow->handle->mark = 1;
9477                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9478                                 const struct rte_flow_action_mark *mark =
9479                                         (const struct rte_flow_action_mark *)
9480                                                 actions->conf;
9481
9482                                 if (flow_dv_convert_action_mark(dev, mark,
9483                                                                 mhdr_res,
9484                                                                 error))
9485                                         return -rte_errno;
9486                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9487                                 break;
9488                         }
9489                         /* Fall-through */
9490                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
9491                         /* Legacy (non-extensive) MARK action. */
9492                         tag_be = mlx5_flow_mark_set
9493                               (((const struct rte_flow_action_mark *)
9494                                (actions->conf))->id);
9495                         MLX5_ASSERT(!handle->dvh.rix_tag);
9496                         if (flow_dv_tag_resource_register(dev, tag_be,
9497                                                           dev_flow, error))
9498                                 return -rte_errno;
9499                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9500                         dev_flow->dv.actions[actions_n++] =
9501                                         dev_flow->dv.tag_resource->action;
9502                         break;
9503                 case RTE_FLOW_ACTION_TYPE_SET_META:
9504                         if (flow_dv_convert_action_set_meta
9505                                 (dev, mhdr_res, attr,
9506                                  (const struct rte_flow_action_set_meta *)
9507                                   actions->conf, error))
9508                                 return -rte_errno;
9509                         action_flags |= MLX5_FLOW_ACTION_SET_META;
9510                         break;
9511                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
9512                         if (flow_dv_convert_action_set_tag
9513                                 (dev, mhdr_res,
9514                                  (const struct rte_flow_action_set_tag *)
9515                                   actions->conf, error))
9516                                 return -rte_errno;
9517                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9518                         break;
9519                 case RTE_FLOW_ACTION_TYPE_DROP:
9520                         action_flags |= MLX5_FLOW_ACTION_DROP;
9521                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
9522                         break;
9523                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9524                         queue = actions->conf;
9525                         rss_desc->queue_num = 1;
9526                         rss_desc->queue[0] = queue->index;
9527                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9528                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9529                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
9530                         num_of_dest++;
9531                         break;
9532                 case RTE_FLOW_ACTION_TYPE_RSS:
9533                         rss = actions->conf;
9534                         memcpy(rss_desc->queue, rss->queue,
9535                                rss->queue_num * sizeof(uint16_t));
9536                         rss_desc->queue_num = rss->queue_num;
9537                         /* NULL RSS key indicates default RSS key. */
9538                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
9539                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
9540                         /*
9541                          * rss->level and rss.types should be set in advance
9542                          * when expanding items for RSS.
9543                          */
9544                         action_flags |= MLX5_FLOW_ACTION_RSS;
9545                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9546                         break;
9547                 case RTE_FLOW_ACTION_TYPE_AGE:
9548                 case RTE_FLOW_ACTION_TYPE_COUNT:
9549                         if (!dev_conf->devx) {
9550                                 return rte_flow_error_set
9551                                               (error, ENOTSUP,
9552                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9553                                                NULL,
9554                                                "count action not supported");
9555                         }
9556                         /* Save information first, will apply later. */
9557                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
9558                                 count = action->conf;
9559                         else
9560                                 age = action->conf;
9561                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9562                         break;
9563                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
9564                         dev_flow->dv.actions[actions_n++] =
9565                                                 priv->sh->pop_vlan_action;
9566                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
9567                         break;
9568                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
9569                         if (!(action_flags &
9570                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
9571                                 flow_dev_get_vlan_info_from_items(items, &vlan);
9572                         vlan.eth_proto = rte_be_to_cpu_16
9573                              ((((const struct rte_flow_action_of_push_vlan *)
9574                                                    actions->conf)->ethertype));
9575                         found_action = mlx5_flow_find_action
9576                                         (actions + 1,
9577                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
9578                         if (found_action)
9579                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9580                         found_action = mlx5_flow_find_action
9581                                         (actions + 1,
9582                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
9583                         if (found_action)
9584                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9585                         if (flow_dv_create_action_push_vlan
9586                                             (dev, attr, &vlan, dev_flow, error))
9587                                 return -rte_errno;
9588                         dev_flow->dv.actions[actions_n++] =
9589                                         dev_flow->dv.push_vlan_res->action;
9590                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
9591                         break;
9592                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
9593                         /* of_vlan_push action handled this action */
9594                         MLX5_ASSERT(action_flags &
9595                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
9596                         break;
9597                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
9598                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
9599                                 break;
9600                         flow_dev_get_vlan_info_from_items(items, &vlan);
9601                         mlx5_update_vlan_vid_pcp(actions, &vlan);
9602                         /* If no VLAN push - this is a modify header action */
9603                         if (flow_dv_convert_action_modify_vlan_vid
9604                                                 (mhdr_res, actions, error))
9605                                 return -rte_errno;
9606                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
9607                         break;
9608                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
9609                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
9610                         if (flow_dv_create_action_l2_encap(dev, actions,
9611                                                            dev_flow,
9612                                                            attr->transfer,
9613                                                            error))
9614                                 return -rte_errno;
9615                         dev_flow->dv.actions[actions_n++] =
9616                                         dev_flow->dv.encap_decap->action;
9617                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9618                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9619                                 sample_act->action_flags |=
9620                                                         MLX5_FLOW_ACTION_ENCAP;
9621                         break;
9622                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
9623                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
9624                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
9625                                                            attr->transfer,
9626                                                            error))
9627                                 return -rte_errno;
9628                         dev_flow->dv.actions[actions_n++] =
9629                                         dev_flow->dv.encap_decap->action;
9630                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9631                         break;
9632                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9633                         /* Handle encap with preceding decap. */
9634                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
9635                                 if (flow_dv_create_action_raw_encap
9636                                         (dev, actions, dev_flow, attr, error))
9637                                         return -rte_errno;
9638                                 dev_flow->dv.actions[actions_n++] =
9639                                         dev_flow->dv.encap_decap->action;
9640                         } else {
9641                                 /* Handle encap without preceding decap. */
9642                                 if (flow_dv_create_action_l2_encap
9643                                     (dev, actions, dev_flow, attr->transfer,
9644                                      error))
9645                                         return -rte_errno;
9646                                 dev_flow->dv.actions[actions_n++] =
9647                                         dev_flow->dv.encap_decap->action;
9648                         }
9649                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9650                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9651                                 sample_act->action_flags |=
9652                                                         MLX5_FLOW_ACTION_ENCAP;
9653                         break;
9654                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
9655                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
9656                                 ;
9657                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
9658                                 if (flow_dv_create_action_l2_decap
9659                                     (dev, dev_flow, attr->transfer, error))
9660                                         return -rte_errno;
9661                                 dev_flow->dv.actions[actions_n++] =
9662                                         dev_flow->dv.encap_decap->action;
9663                         }
9664                         /* If decap is followed by encap, handle it at encap. */
9665                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9666                         break;
9667                 case RTE_FLOW_ACTION_TYPE_JUMP:
9668                         jump_group = ((const struct rte_flow_action_jump *)
9669                                                         action->conf)->group;
9670                         if (dev_flow->external && jump_group <
9671                                         MLX5_MAX_TABLES_EXTERNAL)
9672                                 jump_group *= MLX5_FLOW_TABLE_FACTOR;
9673                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
9674                                                        jump_group,
9675                                                        !!priv->fdb_def_rule,
9676                                                        &table, error);
9677                         if (ret)
9678                                 return ret;
9679                         tbl = flow_dv_tbl_resource_get(dev, table,
9680                                                        attr->egress,
9681                                                        attr->transfer, error);
9682                         if (!tbl)
9683                                 return rte_flow_error_set
9684                                                 (error, errno,
9685                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9686                                                  NULL,
9687                                                  "cannot create jump action.");
9688                         if (flow_dv_jump_tbl_resource_register
9689                             (dev, tbl, dev_flow, error)) {
9690                                 flow_dv_tbl_resource_release(dev, tbl);
9691                                 return rte_flow_error_set
9692                                                 (error, errno,
9693                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9694                                                  NULL,
9695                                                  "cannot create jump action.");
9696                         }
9697                         dev_flow->dv.actions[actions_n++] =
9698                                         dev_flow->dv.jump->action;
9699                         action_flags |= MLX5_FLOW_ACTION_JUMP;
9700                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
9701                         break;
9702                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
9703                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
9704                         if (flow_dv_convert_action_modify_mac
9705                                         (mhdr_res, actions, error))
9706                                 return -rte_errno;
9707                         action_flags |= actions->type ==
9708                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
9709                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
9710                                         MLX5_FLOW_ACTION_SET_MAC_DST;
9711                         break;
9712                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
9713                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
9714                         if (flow_dv_convert_action_modify_ipv4
9715                                         (mhdr_res, actions, error))
9716                                 return -rte_errno;
9717                         action_flags |= actions->type ==
9718                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
9719                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
9720                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
9721                         break;
9722                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
9723                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
9724                         if (flow_dv_convert_action_modify_ipv6
9725                                         (mhdr_res, actions, error))
9726                                 return -rte_errno;
9727                         action_flags |= actions->type ==
9728                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
9729                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
9730                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
9731                         break;
9732                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
9733                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
9734                         if (flow_dv_convert_action_modify_tp
9735                                         (mhdr_res, actions, items,
9736                                          &flow_attr, dev_flow, !!(action_flags &
9737                                          MLX5_FLOW_ACTION_DECAP), error))
9738                                 return -rte_errno;
9739                         action_flags |= actions->type ==
9740                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
9741                                         MLX5_FLOW_ACTION_SET_TP_SRC :
9742                                         MLX5_FLOW_ACTION_SET_TP_DST;
9743                         break;
9744                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
9745                         if (flow_dv_convert_action_modify_dec_ttl
9746                                         (mhdr_res, items, &flow_attr, dev_flow,
9747                                          !!(action_flags &
9748                                          MLX5_FLOW_ACTION_DECAP), error))
9749                                 return -rte_errno;
9750                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
9751                         break;
9752                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
9753                         if (flow_dv_convert_action_modify_ttl
9754                                         (mhdr_res, actions, items, &flow_attr,
9755                                          dev_flow, !!(action_flags &
9756                                          MLX5_FLOW_ACTION_DECAP), error))
9757                                 return -rte_errno;
9758                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
9759                         break;
9760                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
9761                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
9762                         if (flow_dv_convert_action_modify_tcp_seq
9763                                         (mhdr_res, actions, error))
9764                                 return -rte_errno;
9765                         action_flags |= actions->type ==
9766                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
9767                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
9768                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
9769                         break;
9770
9771                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
9772                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
9773                         if (flow_dv_convert_action_modify_tcp_ack
9774                                         (mhdr_res, actions, error))
9775                                 return -rte_errno;
9776                         action_flags |= actions->type ==
9777                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
9778                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
9779                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
9780                         break;
9781                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
9782                         if (flow_dv_convert_action_set_reg
9783                                         (mhdr_res, actions, error))
9784                                 return -rte_errno;
9785                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9786                         break;
9787                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
9788                         if (flow_dv_convert_action_copy_mreg
9789                                         (dev, mhdr_res, actions, error))
9790                                 return -rte_errno;
9791                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9792                         break;
9793                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
9794                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
9795                         dev_flow->handle->fate_action =
9796                                         MLX5_FLOW_FATE_DEFAULT_MISS;
9797                         break;
9798                 case RTE_FLOW_ACTION_TYPE_METER:
9799                         mtr = actions->conf;
9800                         if (!flow->meter) {
9801                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
9802                                                             attr, error);
9803                                 if (!fm)
9804                                         return rte_flow_error_set(error,
9805                                                 rte_errno,
9806                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9807                                                 NULL,
9808                                                 "meter not found "
9809                                                 "or invalid parameters");
9810                                 flow->meter = fm->idx;
9811                         }
9812                         /* Set the meter action. */
9813                         if (!fm) {
9814                                 fm = mlx5_ipool_get(priv->sh->ipool
9815                                                 [MLX5_IPOOL_MTR], flow->meter);
9816                                 if (!fm)
9817                                         return rte_flow_error_set(error,
9818                                                 rte_errno,
9819                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9820                                                 NULL,
9821                                                 "meter not found "
9822                                                 "or invalid parameters");
9823                         }
9824                         dev_flow->dv.actions[actions_n++] =
9825                                 fm->mfts->meter_action;
9826                         action_flags |= MLX5_FLOW_ACTION_METER;
9827                         break;
9828                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
9829                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
9830                                                               actions, error))
9831                                 return -rte_errno;
9832                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
9833                         break;
9834                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
9835                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
9836                                                               actions, error))
9837                                 return -rte_errno;
9838                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
9839                         break;
9840                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
9841                         sample_act_pos = actions_n;
9842                         ret = flow_dv_translate_action_sample(dev,
9843                                                               actions,
9844                                                               dev_flow, attr,
9845                                                               &num_of_dest,
9846                                                               sample_actions,
9847                                                               &sample_res,
9848                                                               error);
9849                         if (ret < 0)
9850                                 return ret;
9851                         actions_n++;
9852                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
9853                         /* put encap action into group if work with port id */
9854                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
9855                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
9856                                 sample_act->action_flags |=
9857                                                         MLX5_FLOW_ACTION_ENCAP;
9858                         break;
9859                 case RTE_FLOW_ACTION_TYPE_END:
9860                         actions_end = true;
9861                         if (mhdr_res->actions_num) {
9862                                 /* create modify action if needed. */
9863                                 if (flow_dv_modify_hdr_resource_register
9864                                         (dev, mhdr_res, dev_flow, error))
9865                                         return -rte_errno;
9866                                 dev_flow->dv.actions[modify_action_position] =
9867                                         handle->dvh.modify_hdr->action;
9868                         }
9869                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
9870                                 flow->counter =
9871                                         flow_dv_translate_create_counter(dev,
9872                                                 dev_flow, count, age);
9873
9874                                 if (!flow->counter)
9875                                         return rte_flow_error_set
9876                                                 (error, rte_errno,
9877                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9878                                                 NULL,
9879                                                 "cannot create counter"
9880                                                 " object.");
9881                                 dev_flow->dv.actions[actions_n] =
9882                                           (flow_dv_counter_get_by_idx(dev,
9883                                           flow->counter, NULL))->action;
9884                                 actions_n++;
9885                         }
9886                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
9887                                 ret = flow_dv_create_action_sample(dev,
9888                                                           dev_flow, attr,
9889                                                           num_of_dest,
9890                                                           &sample_res,
9891                                                           &mdest_res,
9892                                                           sample_actions,
9893                                                           action_flags,
9894                                                           error);
9895                                 if (ret < 0)
9896                                         return rte_flow_error_set
9897                                                 (error, rte_errno,
9898                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9899                                                 NULL,
9900                                                 "cannot create sample action");
9901                                 if (num_of_dest > 1) {
9902                                         dev_flow->dv.actions[sample_act_pos] =
9903                                         dev_flow->dv.dest_array_res->action;
9904                                 } else {
9905                                         dev_flow->dv.actions[sample_act_pos] =
9906                                         dev_flow->dv.sample_res->verbs_action;
9907                                 }
9908                         }
9909                         break;
9910                 default:
9911                         break;
9912                 }
9913                 if (mhdr_res->actions_num &&
9914                     modify_action_position == UINT32_MAX)
9915                         modify_action_position = actions_n++;
9916         }
9917         /*
9918          * For multiple destination (sample action with ratio=1), the encap
9919          * action and port id action will be combined into group action.
9920          * So need remove the original these actions in the flow and only
9921          * use the sample action instead of.
9922          */
9923         if (num_of_dest > 1 && sample_act->dr_port_id_action) {
9924                 int i;
9925                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9926
9927                 for (i = 0; i < actions_n; i++) {
9928                         if ((sample_act->dr_encap_action &&
9929                                 sample_act->dr_encap_action ==
9930                                 dev_flow->dv.actions[i]) ||
9931                                 (sample_act->dr_port_id_action &&
9932                                 sample_act->dr_port_id_action ==
9933                                 dev_flow->dv.actions[i]))
9934                                 continue;
9935                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
9936                 }
9937                 memcpy((void *)dev_flow->dv.actions,
9938                                 (void *)temp_actions,
9939                                 tmp_actions_n * sizeof(void *));
9940                 actions_n = tmp_actions_n;
9941         }
9942         dev_flow->dv.actions_n = actions_n;
9943         dev_flow->act_flags = action_flags;
9944         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
9945                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
9946                 int item_type = items->type;
9947
9948                 if (!mlx5_flow_os_item_supported(item_type))
9949                         return rte_flow_error_set(error, ENOTSUP,
9950                                                   RTE_FLOW_ERROR_TYPE_ITEM,
9951                                                   NULL, "item not supported");
9952                 switch (item_type) {
9953                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
9954                         flow_dv_translate_item_port_id(dev, match_mask,
9955                                                        match_value, items);
9956                         last_item = MLX5_FLOW_ITEM_PORT_ID;
9957                         break;
9958                 case RTE_FLOW_ITEM_TYPE_ETH:
9959                         flow_dv_translate_item_eth(match_mask, match_value,
9960                                                    items, tunnel,
9961                                                    dev_flow->dv.group);
9962                         matcher.priority = action_flags &
9963                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
9964                                         !dev_flow->external ?
9965                                         MLX5_PRIORITY_MAP_L3 :
9966                                         MLX5_PRIORITY_MAP_L2;
9967                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
9968                                              MLX5_FLOW_LAYER_OUTER_L2;
9969                         break;
9970                 case RTE_FLOW_ITEM_TYPE_VLAN:
9971                         flow_dv_translate_item_vlan(dev_flow,
9972                                                     match_mask, match_value,
9973                                                     items, tunnel,
9974                                                     dev_flow->dv.group);
9975                         matcher.priority = MLX5_PRIORITY_MAP_L2;
9976                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
9977                                               MLX5_FLOW_LAYER_INNER_VLAN) :
9978                                              (MLX5_FLOW_LAYER_OUTER_L2 |
9979                                               MLX5_FLOW_LAYER_OUTER_VLAN);
9980                         break;
9981                 case RTE_FLOW_ITEM_TYPE_IPV4:
9982                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9983                                                   &item_flags, &tunnel);
9984                         flow_dv_translate_item_ipv4(match_mask, match_value,
9985                                                     items, item_flags, tunnel,
9986                                                     dev_flow->dv.group);
9987                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9988                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
9989                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
9990                         if (items->mask != NULL &&
9991                             ((const struct rte_flow_item_ipv4 *)
9992                              items->mask)->hdr.next_proto_id) {
9993                                 next_protocol =
9994                                         ((const struct rte_flow_item_ipv4 *)
9995                                          (items->spec))->hdr.next_proto_id;
9996                                 next_protocol &=
9997                                         ((const struct rte_flow_item_ipv4 *)
9998                                          (items->mask))->hdr.next_proto_id;
9999                         } else {
10000                                 /* Reset for inner layer. */
10001                                 next_protocol = 0xff;
10002                         }
10003                         break;
10004                 case RTE_FLOW_ITEM_TYPE_IPV6:
10005                         mlx5_flow_tunnel_ip_check(items, next_protocol,
10006                                                   &item_flags, &tunnel);
10007                         flow_dv_translate_item_ipv6(match_mask, match_value,
10008                                                     items, item_flags, tunnel,
10009                                                     dev_flow->dv.group);
10010                         matcher.priority = MLX5_PRIORITY_MAP_L3;
10011                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
10012                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
10013                         if (items->mask != NULL &&
10014                             ((const struct rte_flow_item_ipv6 *)
10015                              items->mask)->hdr.proto) {
10016                                 next_protocol =
10017                                         ((const struct rte_flow_item_ipv6 *)
10018                                          items->spec)->hdr.proto;
10019                                 next_protocol &=
10020                                         ((const struct rte_flow_item_ipv6 *)
10021                                          items->mask)->hdr.proto;
10022                         } else {
10023                                 /* Reset for inner layer. */
10024                                 next_protocol = 0xff;
10025                         }
10026                         break;
10027                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
10028                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
10029                                                              match_value,
10030                                                              items, tunnel);
10031                         last_item = tunnel ?
10032                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
10033                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
10034                         if (items->mask != NULL &&
10035                             ((const struct rte_flow_item_ipv6_frag_ext *)
10036                              items->mask)->hdr.next_header) {
10037                                 next_protocol =
10038                                 ((const struct rte_flow_item_ipv6_frag_ext *)
10039                                  items->spec)->hdr.next_header;
10040                                 next_protocol &=
10041                                 ((const struct rte_flow_item_ipv6_frag_ext *)
10042                                  items->mask)->hdr.next_header;
10043                         } else {
10044                                 /* Reset for inner layer. */
10045                                 next_protocol = 0xff;
10046                         }
10047                         break;
10048                 case RTE_FLOW_ITEM_TYPE_TCP:
10049                         flow_dv_translate_item_tcp(match_mask, match_value,
10050                                                    items, tunnel);
10051                         matcher.priority = MLX5_PRIORITY_MAP_L4;
10052                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
10053                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
10054                         break;
10055                 case RTE_FLOW_ITEM_TYPE_UDP:
10056                         flow_dv_translate_item_udp(match_mask, match_value,
10057                                                    items, tunnel);
10058                         matcher.priority = MLX5_PRIORITY_MAP_L4;
10059                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
10060                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
10061                         break;
10062                 case RTE_FLOW_ITEM_TYPE_GRE:
10063                         flow_dv_translate_item_gre(match_mask, match_value,
10064                                                    items, tunnel);
10065                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10066                         last_item = MLX5_FLOW_LAYER_GRE;
10067                         break;
10068                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
10069                         flow_dv_translate_item_gre_key(match_mask,
10070                                                        match_value, items);
10071                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
10072                         break;
10073                 case RTE_FLOW_ITEM_TYPE_NVGRE:
10074                         flow_dv_translate_item_nvgre(match_mask, match_value,
10075                                                      items, tunnel);
10076                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10077                         last_item = MLX5_FLOW_LAYER_GRE;
10078                         break;
10079                 case RTE_FLOW_ITEM_TYPE_VXLAN:
10080                         flow_dv_translate_item_vxlan(match_mask, match_value,
10081                                                      items, tunnel);
10082                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10083                         last_item = MLX5_FLOW_LAYER_VXLAN;
10084                         break;
10085                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
10086                         flow_dv_translate_item_vxlan_gpe(match_mask,
10087                                                          match_value, items,
10088                                                          tunnel);
10089                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10090                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
10091                         break;
10092                 case RTE_FLOW_ITEM_TYPE_GENEVE:
10093                         flow_dv_translate_item_geneve(match_mask, match_value,
10094                                                       items, tunnel);
10095                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10096                         last_item = MLX5_FLOW_LAYER_GENEVE;
10097                         break;
10098                 case RTE_FLOW_ITEM_TYPE_MPLS:
10099                         flow_dv_translate_item_mpls(match_mask, match_value,
10100                                                     items, last_item, tunnel);
10101                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10102                         last_item = MLX5_FLOW_LAYER_MPLS;
10103                         break;
10104                 case RTE_FLOW_ITEM_TYPE_MARK:
10105                         flow_dv_translate_item_mark(dev, match_mask,
10106                                                     match_value, items);
10107                         last_item = MLX5_FLOW_ITEM_MARK;
10108                         break;
10109                 case RTE_FLOW_ITEM_TYPE_META:
10110                         flow_dv_translate_item_meta(dev, match_mask,
10111                                                     match_value, attr, items);
10112                         last_item = MLX5_FLOW_ITEM_METADATA;
10113                         break;
10114                 case RTE_FLOW_ITEM_TYPE_ICMP:
10115                         flow_dv_translate_item_icmp(match_mask, match_value,
10116                                                     items, tunnel);
10117                         last_item = MLX5_FLOW_LAYER_ICMP;
10118                         break;
10119                 case RTE_FLOW_ITEM_TYPE_ICMP6:
10120                         flow_dv_translate_item_icmp6(match_mask, match_value,
10121                                                       items, tunnel);
10122                         last_item = MLX5_FLOW_LAYER_ICMP6;
10123                         break;
10124                 case RTE_FLOW_ITEM_TYPE_TAG:
10125                         flow_dv_translate_item_tag(dev, match_mask,
10126                                                    match_value, items);
10127                         last_item = MLX5_FLOW_ITEM_TAG;
10128                         break;
10129                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
10130                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
10131                                                         match_value, items);
10132                         last_item = MLX5_FLOW_ITEM_TAG;
10133                         break;
10134                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
10135                         flow_dv_translate_item_tx_queue(dev, match_mask,
10136                                                         match_value,
10137                                                         items);
10138                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
10139                         break;
10140                 case RTE_FLOW_ITEM_TYPE_GTP:
10141                         flow_dv_translate_item_gtp(match_mask, match_value,
10142                                                    items, tunnel);
10143                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10144                         last_item = MLX5_FLOW_LAYER_GTP;
10145                         break;
10146                 case RTE_FLOW_ITEM_TYPE_ECPRI:
10147                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
10148                                 /* Create it only the first time to be used. */
10149                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
10150                                 if (ret)
10151                                         return rte_flow_error_set
10152                                                 (error, -ret,
10153                                                 RTE_FLOW_ERROR_TYPE_ITEM,
10154                                                 NULL,
10155                                                 "cannot create eCPRI parser");
10156                         }
10157                         /* Adjust the length matcher and device flow value. */
10158                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
10159                         dev_flow->dv.value.size =
10160                                         MLX5_ST_SZ_BYTES(fte_match_param);
10161                         flow_dv_translate_item_ecpri(dev, match_mask,
10162                                                      match_value, items);
10163                         /* No other protocol should follow eCPRI layer. */
10164                         last_item = MLX5_FLOW_LAYER_ECPRI;
10165                         break;
10166                 default:
10167                         break;
10168                 }
10169                 item_flags |= last_item;
10170         }
10171         /*
10172          * When E-Switch mode is enabled, we have two cases where we need to
10173          * set the source port manually.
10174          * The first one, is in case of Nic steering rule, and the second is
10175          * E-Switch rule where no port_id item was found. In both cases
10176          * the source port is set according the current port in use.
10177          */
10178         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
10179             (priv->representor || priv->master)) {
10180                 if (flow_dv_translate_item_port_id(dev, match_mask,
10181                                                    match_value, NULL))
10182                         return -rte_errno;
10183         }
10184 #ifdef RTE_LIBRTE_MLX5_DEBUG
10185         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
10186                                               dev_flow->dv.value.buf));
10187 #endif
10188         /*
10189          * Layers may be already initialized from prefix flow if this dev_flow
10190          * is the suffix flow.
10191          */
10192         handle->layers |= item_flags;
10193         if (action_flags & MLX5_FLOW_ACTION_RSS)
10194                 flow_dv_hashfields_set(dev_flow, rss_desc);
10195         /* Register matcher. */
10196         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
10197                                     matcher.mask.size);
10198         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
10199                                                      matcher.priority);
10200         /* reserved field no needs to be set to 0 here. */
10201         tbl_key.domain = attr->transfer;
10202         tbl_key.direction = attr->egress;
10203         tbl_key.table_id = dev_flow->dv.group;
10204         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
10205                 return -rte_errno;
10206         return 0;
10207 }
10208
10209 /**
10210  * Apply the flow to the NIC, lock free,
10211  * (mutex should be acquired by caller).
10212  *
10213  * @param[in] dev
10214  *   Pointer to the Ethernet device structure.
10215  * @param[in, out] flow
10216  *   Pointer to flow structure.
10217  * @param[out] error
10218  *   Pointer to error structure.
10219  *
10220  * @return
10221  *   0 on success, a negative errno value otherwise and rte_errno is set.
10222  */
10223 static int
10224 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
10225                 struct rte_flow_error *error)
10226 {
10227         struct mlx5_flow_dv_workspace *dv;
10228         struct mlx5_flow_handle *dh;
10229         struct mlx5_flow_handle_dv *dv_h;
10230         struct mlx5_flow *dev_flow;
10231         struct mlx5_priv *priv = dev->data->dev_private;
10232         uint32_t handle_idx;
10233         int n;
10234         int err;
10235         int idx;
10236
10237         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
10238                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
10239                 dv = &dev_flow->dv;
10240                 dh = dev_flow->handle;
10241                 dv_h = &dh->dvh;
10242                 n = dv->actions_n;
10243                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10244                         if (dv->transfer) {
10245                                 dv->actions[n++] = priv->sh->esw_drop_action;
10246                         } else {
10247                                 struct mlx5_hrxq *drop_hrxq;
10248                                 drop_hrxq = mlx5_drop_action_create(dev);
10249                                 if (!drop_hrxq) {
10250                                         rte_flow_error_set
10251                                                 (error, errno,
10252                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10253                                                  NULL,
10254                                                  "cannot get drop hash queue");
10255                                         goto error;
10256                                 }
10257                                 /*
10258                                  * Drop queues will be released by the specify
10259                                  * mlx5_drop_action_destroy() function. Assign
10260                                  * the special index to hrxq to mark the queue
10261                                  * has been allocated.
10262                                  */
10263                                 dh->rix_hrxq = UINT32_MAX;
10264                                 dv->actions[n++] = drop_hrxq->action;
10265                         }
10266                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
10267                            !dv_h->rix_sample && !dv_h->rix_dest_array) {
10268                         struct mlx5_hrxq *hrxq;
10269                         uint32_t hrxq_idx;
10270                         struct mlx5_flow_rss_desc *rss_desc =
10271                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
10272                                 [!!priv->flow_nested_idx];
10273
10274                         MLX5_ASSERT(rss_desc->queue_num);
10275                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
10276                                                  MLX5_RSS_HASH_KEY_LEN,
10277                                                  dev_flow->hash_fields,
10278                                                  rss_desc->queue,
10279                                                  rss_desc->queue_num);
10280                         if (!hrxq_idx) {
10281                                 hrxq_idx = mlx5_hrxq_new
10282                                                 (dev, rss_desc->key,
10283                                                  MLX5_RSS_HASH_KEY_LEN,
10284                                                  dev_flow->hash_fields,
10285                                                  rss_desc->queue,
10286                                                  rss_desc->queue_num,
10287                                                  !!(dh->layers &
10288                                                  MLX5_FLOW_LAYER_TUNNEL));
10289                         }
10290                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10291                                               hrxq_idx);
10292                         if (!hrxq) {
10293                                 rte_flow_error_set
10294                                         (error, rte_errno,
10295                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10296                                          "cannot get hash queue");
10297                                 goto error;
10298                         }
10299                         dh->rix_hrxq = hrxq_idx;
10300                         dv->actions[n++] = hrxq->action;
10301                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
10302                         if (flow_dv_default_miss_resource_register
10303                                         (dev, error)) {
10304                                 rte_flow_error_set
10305                                         (error, rte_errno,
10306                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10307                                          "cannot create default miss resource");
10308                                 goto error_default_miss;
10309                         }
10310                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
10311                         dv->actions[n++] = priv->sh->default_miss.action;
10312                 }
10313                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
10314                                                (void *)&dv->value, n,
10315                                                dv->actions, &dh->drv_flow);
10316                 if (err) {
10317                         rte_flow_error_set(error, errno,
10318                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10319                                            NULL,
10320                                            "hardware refuses to create flow");
10321                         goto error;
10322                 }
10323                 if (priv->vmwa_context &&
10324                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
10325                         /*
10326                          * The rule contains the VLAN pattern.
10327                          * For VF we are going to create VLAN
10328                          * interface to make hypervisor set correct
10329                          * e-Switch vport context.
10330                          */
10331                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
10332                 }
10333         }
10334         return 0;
10335 error:
10336         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10337                 flow_dv_default_miss_resource_release(dev);
10338 error_default_miss:
10339         err = rte_errno; /* Save rte_errno before cleanup. */
10340         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
10341                        handle_idx, dh, next) {
10342                 /* hrxq is union, don't clear it if the flag is not set. */
10343                 if (dh->rix_hrxq) {
10344                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10345                                 mlx5_drop_action_destroy(dev);
10346                                 dh->rix_hrxq = 0;
10347                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
10348                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
10349                                 dh->rix_hrxq = 0;
10350                         }
10351                 }
10352                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10353                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10354         }
10355         rte_errno = err; /* Restore rte_errno. */
10356         return -rte_errno;
10357 }
10358
10359 /**
10360  * Release the flow matcher.
10361  *
10362  * @param dev
10363  *   Pointer to Ethernet device.
10364  * @param handle
10365  *   Pointer to mlx5_flow_handle.
10366  *
10367  * @return
10368  *   1 while a reference on it exists, 0 when freed.
10369  */
10370 static int
10371 flow_dv_matcher_release(struct rte_eth_dev *dev,
10372                         struct mlx5_flow_handle *handle)
10373 {
10374         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
10375
10376         MLX5_ASSERT(matcher->matcher_object);
10377         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
10378                 dev->data->port_id, (void *)matcher,
10379                 rte_atomic32_read(&matcher->refcnt));
10380         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
10381                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10382                            (matcher->matcher_object));
10383                 LIST_REMOVE(matcher, next);
10384                 /* table ref-- in release interface. */
10385                 flow_dv_tbl_resource_release(dev, matcher->tbl);
10386                 mlx5_free(matcher);
10387                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
10388                         dev->data->port_id, (void *)matcher);
10389                 return 0;
10390         }
10391         return 1;
10392 }
10393
10394 /**
10395  * Release an encap/decap resource.
10396  *
10397  * @param dev
10398  *   Pointer to Ethernet device.
10399  * @param encap_decap_idx
10400  *   Index of encap decap resource.
10401  *
10402  * @return
10403  *   1 while a reference on it exists, 0 when freed.
10404  */
10405 static int
10406 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
10407                                      uint32_t encap_decap_idx)
10408 {
10409         struct mlx5_priv *priv = dev->data->dev_private;
10410         uint32_t idx = encap_decap_idx;
10411         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
10412
10413         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
10414                          idx);
10415         if (!cache_resource)
10416                 return 0;
10417         MLX5_ASSERT(cache_resource->action);
10418         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
10419                 (void *)cache_resource,
10420                 rte_atomic32_read(&cache_resource->refcnt));
10421         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10422                 claim_zero(mlx5_flow_os_destroy_flow_action
10423                                                 (cache_resource->action));
10424                 mlx5_hlist_remove(priv->sh->encaps_decaps,
10425                                   &cache_resource->entry);
10426                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
10427                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
10428                         (void *)cache_resource);
10429                 return 0;
10430         }
10431         return 1;
10432 }
10433
10434 /**
10435  * Release an jump to table action resource.
10436  *
10437  * @param dev
10438  *   Pointer to Ethernet device.
10439  * @param handle
10440  *   Pointer to mlx5_flow_handle.
10441  *
10442  * @return
10443  *   1 while a reference on it exists, 0 when freed.
10444  */
10445 static int
10446 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
10447                                   struct mlx5_flow_handle *handle)
10448 {
10449         struct mlx5_priv *priv = dev->data->dev_private;
10450         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
10451         struct mlx5_flow_tbl_data_entry *tbl_data;
10452
10453         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
10454                              handle->rix_jump);
10455         if (!tbl_data)
10456                 return 0;
10457         cache_resource = &tbl_data->jump;
10458         MLX5_ASSERT(cache_resource->action);
10459         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
10460                 (void *)cache_resource,
10461                 rte_atomic32_read(&cache_resource->refcnt));
10462         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10463                 claim_zero(mlx5_flow_os_destroy_flow_action
10464                                                 (cache_resource->action));
10465                 /* jump action memory free is inside the table release. */
10466                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
10467                 DRV_LOG(DEBUG, "jump table resource %p: removed",
10468                         (void *)cache_resource);
10469                 return 0;
10470         }
10471         return 1;
10472 }
10473
10474 /**
10475  * Release a default miss resource.
10476  *
10477  * @param dev
10478  *   Pointer to Ethernet device.
10479  * @return
10480  *   1 while a reference on it exists, 0 when freed.
10481  */
10482 static int
10483 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
10484 {
10485         struct mlx5_priv *priv = dev->data->dev_private;
10486         struct mlx5_dev_ctx_shared *sh = priv->sh;
10487         struct mlx5_flow_default_miss_resource *cache_resource =
10488                         &sh->default_miss;
10489
10490         MLX5_ASSERT(cache_resource->action);
10491         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
10492                         (void *)cache_resource->action,
10493                         rte_atomic32_read(&cache_resource->refcnt));
10494         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10495                 claim_zero(mlx5_glue->destroy_flow_action
10496                                 (cache_resource->action));
10497                 DRV_LOG(DEBUG, "default miss resource %p: removed",
10498                                 (void *)cache_resource->action);
10499                 return 0;
10500         }
10501         return 1;
10502 }
10503
10504 /**
10505  * Release a modify-header resource.
10506  *
10507  * @param dev
10508  *   Pointer to Ethernet device.
10509  * @param handle
10510  *   Pointer to mlx5_flow_handle.
10511  *
10512  * @return
10513  *   1 while a reference on it exists, 0 when freed.
10514  */
10515 static int
10516 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
10517                                     struct mlx5_flow_handle *handle)
10518 {
10519         struct mlx5_priv *priv = dev->data->dev_private;
10520         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
10521                                                         handle->dvh.modify_hdr;
10522
10523         MLX5_ASSERT(cache_resource->action);
10524         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
10525                 (void *)cache_resource,
10526                 rte_atomic32_read(&cache_resource->refcnt));
10527         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10528                 claim_zero(mlx5_flow_os_destroy_flow_action
10529                                                 (cache_resource->action));
10530                 mlx5_hlist_remove(priv->sh->modify_cmds,
10531                                   &cache_resource->entry);
10532                 mlx5_free(cache_resource);
10533                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
10534                         (void *)cache_resource);
10535                 return 0;
10536         }
10537         return 1;
10538 }
10539
10540 /**
10541  * Release port ID action resource.
10542  *
10543  * @param dev
10544  *   Pointer to Ethernet device.
10545  * @param handle
10546  *   Pointer to mlx5_flow_handle.
10547  *
10548  * @return
10549  *   1 while a reference on it exists, 0 when freed.
10550  */
10551 static int
10552 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
10553                                         uint32_t port_id)
10554 {
10555         struct mlx5_priv *priv = dev->data->dev_private;
10556         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
10557         uint32_t idx = port_id;
10558
10559         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10560                                         idx);
10561         if (!cache_resource)
10562                 return 0;
10563         MLX5_ASSERT(cache_resource->action);
10564         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
10565                 (void *)cache_resource,
10566                 rte_atomic32_read(&cache_resource->refcnt));
10567         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10568                 claim_zero(mlx5_flow_os_destroy_flow_action
10569                                                 (cache_resource->action));
10570                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10571                              &priv->sh->port_id_action_list, idx,
10572                              cache_resource, next);
10573                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
10574                 DRV_LOG(DEBUG, "port id action resource %p: removed",
10575                         (void *)cache_resource);
10576                 return 0;
10577         }
10578         return 1;
10579 }
10580
10581 /**
10582  * Release push vlan action resource.
10583  *
10584  * @param dev
10585  *   Pointer to Ethernet device.
10586  * @param handle
10587  *   Pointer to mlx5_flow_handle.
10588  *
10589  * @return
10590  *   1 while a reference on it exists, 0 when freed.
10591  */
10592 static int
10593 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
10594                                           struct mlx5_flow_handle *handle)
10595 {
10596         struct mlx5_priv *priv = dev->data->dev_private;
10597         uint32_t idx = handle->dvh.rix_push_vlan;
10598         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
10599
10600         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10601                                         idx);
10602         if (!cache_resource)
10603                 return 0;
10604         MLX5_ASSERT(cache_resource->action);
10605         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
10606                 (void *)cache_resource,
10607                 rte_atomic32_read(&cache_resource->refcnt));
10608         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10609                 claim_zero(mlx5_flow_os_destroy_flow_action
10610                                                 (cache_resource->action));
10611                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10612                              &priv->sh->push_vlan_action_list, idx,
10613                              cache_resource, next);
10614                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
10615                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
10616                         (void *)cache_resource);
10617                 return 0;
10618         }
10619         return 1;
10620 }
10621
10622 /**
10623  * Release the fate resource.
10624  *
10625  * @param dev
10626  *   Pointer to Ethernet device.
10627  * @param handle
10628  *   Pointer to mlx5_flow_handle.
10629  */
10630 static void
10631 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
10632                                struct mlx5_flow_handle *handle)
10633 {
10634         if (!handle->rix_fate)
10635                 return;
10636         switch (handle->fate_action) {
10637         case MLX5_FLOW_FATE_DROP:
10638                 mlx5_drop_action_destroy(dev);
10639                 break;
10640         case MLX5_FLOW_FATE_QUEUE:
10641                 mlx5_hrxq_release(dev, handle->rix_hrxq);
10642                 break;
10643         case MLX5_FLOW_FATE_JUMP:
10644                 flow_dv_jump_tbl_resource_release(dev, handle);
10645                 break;
10646         case MLX5_FLOW_FATE_PORT_ID:
10647                 flow_dv_port_id_action_resource_release(dev,
10648                                 handle->rix_port_id_action);
10649                 break;
10650         case MLX5_FLOW_FATE_DEFAULT_MISS:
10651                 flow_dv_default_miss_resource_release(dev);
10652                 break;
10653         default:
10654                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
10655                 break;
10656         }
10657         handle->rix_fate = 0;
10658 }
10659
10660 /**
10661  * Release an sample resource.
10662  *
10663  * @param dev
10664  *   Pointer to Ethernet device.
10665  * @param handle
10666  *   Pointer to mlx5_flow_handle.
10667  *
10668  * @return
10669  *   1 while a reference on it exists, 0 when freed.
10670  */
10671 static int
10672 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
10673                                      struct mlx5_flow_handle *handle)
10674 {
10675         struct mlx5_priv *priv = dev->data->dev_private;
10676         uint32_t idx = handle->dvh.rix_sample;
10677         struct mlx5_flow_dv_sample_resource *cache_resource;
10678
10679         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10680                          idx);
10681         if (!cache_resource)
10682                 return 0;
10683         MLX5_ASSERT(cache_resource->verbs_action);
10684         DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
10685                 (void *)cache_resource,
10686                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10687         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10688                                __ATOMIC_RELAXED) == 0) {
10689                 if (cache_resource->verbs_action)
10690                         claim_zero(mlx5_glue->destroy_flow_action
10691                                         (cache_resource->verbs_action));
10692                 if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10693                         if (cache_resource->default_miss)
10694                                 claim_zero(mlx5_glue->destroy_flow_action
10695                                   (cache_resource->default_miss));
10696                 }
10697                 if (cache_resource->normal_path_tbl)
10698                         flow_dv_tbl_resource_release(dev,
10699                                 cache_resource->normal_path_tbl);
10700         }
10701         if (cache_resource->sample_idx.rix_hrxq &&
10702                 !mlx5_hrxq_release(dev,
10703                         cache_resource->sample_idx.rix_hrxq))
10704                 cache_resource->sample_idx.rix_hrxq = 0;
10705         if (cache_resource->sample_idx.rix_tag &&
10706                 !flow_dv_tag_release(dev,
10707                         cache_resource->sample_idx.rix_tag))
10708                 cache_resource->sample_idx.rix_tag = 0;
10709         if (cache_resource->sample_idx.cnt) {
10710                 flow_dv_counter_release(dev,
10711                         cache_resource->sample_idx.cnt);
10712                 cache_resource->sample_idx.cnt = 0;
10713         }
10714         if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
10715                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10716                              &priv->sh->sample_action_list, idx,
10717                              cache_resource, next);
10718                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10719                 DRV_LOG(DEBUG, "sample resource %p: removed",
10720                         (void *)cache_resource);
10721                 return 0;
10722         }
10723         return 1;
10724 }
10725
10726 /**
10727  * Release an destination array resource.
10728  *
10729  * @param dev
10730  *   Pointer to Ethernet device.
10731  * @param handle
10732  *   Pointer to mlx5_flow_handle.
10733  *
10734  * @return
10735  *   1 while a reference on it exists, 0 when freed.
10736  */
10737 static int
10738 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
10739                                      struct mlx5_flow_handle *handle)
10740 {
10741         struct mlx5_priv *priv = dev->data->dev_private;
10742         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10743         struct mlx5_flow_sub_actions_idx *mdest_act_res;
10744         uint32_t idx = handle->dvh.rix_dest_array;
10745         uint32_t i = 0;
10746
10747         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10748                          idx);
10749         if (!cache_resource)
10750                 return 0;
10751         MLX5_ASSERT(cache_resource->action);
10752         DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
10753                 (void *)cache_resource,
10754                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10755         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10756                                __ATOMIC_RELAXED) == 0) {
10757                 if (cache_resource->action)
10758                         claim_zero(mlx5_glue->destroy_flow_action
10759                                                 (cache_resource->action));
10760                 for (; i < cache_resource->num_of_dest; i++) {
10761                         mdest_act_res = &cache_resource->sample_idx[i];
10762                         if (mdest_act_res->rix_hrxq) {
10763                                 mlx5_hrxq_release(dev,
10764                                         mdest_act_res->rix_hrxq);
10765                                 mdest_act_res->rix_hrxq = 0;
10766                         }
10767                         if (mdest_act_res->rix_encap_decap) {
10768                                 flow_dv_encap_decap_resource_release(dev,
10769                                         mdest_act_res->rix_encap_decap);
10770                                 mdest_act_res->rix_encap_decap = 0;
10771                         }
10772                         if (mdest_act_res->rix_port_id_action) {
10773                                 flow_dv_port_id_action_resource_release(dev,
10774                                         mdest_act_res->rix_port_id_action);
10775                                 mdest_act_res->rix_port_id_action = 0;
10776                         }
10777                         if (mdest_act_res->rix_tag) {
10778                                 flow_dv_tag_release(dev,
10779                                         mdest_act_res->rix_tag);
10780                                 mdest_act_res->rix_tag = 0;
10781                         }
10782                 }
10783                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10784                              &priv->sh->dest_array_list, idx,
10785                              cache_resource, next);
10786                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
10787                 DRV_LOG(DEBUG, "destination array resource %p: removed",
10788                         (void *)cache_resource);
10789                 return 0;
10790         }
10791         return 1;
10792 }
10793
10794 /**
10795  * Remove the flow from the NIC but keeps it in memory.
10796  * Lock free, (mutex should be acquired by caller).
10797  *
10798  * @param[in] dev
10799  *   Pointer to Ethernet device.
10800  * @param[in, out] flow
10801  *   Pointer to flow structure.
10802  */
10803 static void
10804 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
10805 {
10806         struct mlx5_flow_handle *dh;
10807         uint32_t handle_idx;
10808         struct mlx5_priv *priv = dev->data->dev_private;
10809
10810         if (!flow)
10811                 return;
10812         handle_idx = flow->dev_handles;
10813         while (handle_idx) {
10814                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10815                                     handle_idx);
10816                 if (!dh)
10817                         return;
10818                 if (dh->drv_flow) {
10819                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
10820                         dh->drv_flow = NULL;
10821                 }
10822                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
10823                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
10824                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10825                         flow_dv_fate_resource_release(dev, dh);
10826                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10827                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10828                 handle_idx = dh->next.next;
10829         }
10830 }
10831
10832 /**
10833  * Remove the flow from the NIC and the memory.
10834  * Lock free, (mutex should be acquired by caller).
10835  *
10836  * @param[in] dev
10837  *   Pointer to the Ethernet device structure.
10838  * @param[in, out] flow
10839  *   Pointer to flow structure.
10840  */
10841 static void
10842 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
10843 {
10844         struct mlx5_flow_handle *dev_handle;
10845         struct mlx5_priv *priv = dev->data->dev_private;
10846
10847         if (!flow)
10848                 return;
10849         __flow_dv_remove(dev, flow);
10850         if (flow->counter) {
10851                 flow_dv_counter_release(dev, flow->counter);
10852                 flow->counter = 0;
10853         }
10854         if (flow->meter) {
10855                 struct mlx5_flow_meter *fm;
10856
10857                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
10858                                     flow->meter);
10859                 if (fm)
10860                         mlx5_flow_meter_detach(fm);
10861                 flow->meter = 0;
10862         }
10863         while (flow->dev_handles) {
10864                 uint32_t tmp_idx = flow->dev_handles;
10865
10866                 dev_handle = mlx5_ipool_get(priv->sh->ipool
10867                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
10868                 if (!dev_handle)
10869                         return;
10870                 flow->dev_handles = dev_handle->next.next;
10871                 if (dev_handle->dvh.matcher)
10872                         flow_dv_matcher_release(dev, dev_handle);
10873                 if (dev_handle->dvh.rix_sample)
10874                         flow_dv_sample_resource_release(dev, dev_handle);
10875                 if (dev_handle->dvh.rix_dest_array)
10876                         flow_dv_dest_array_resource_release(dev, dev_handle);
10877                 if (dev_handle->dvh.rix_encap_decap)
10878                         flow_dv_encap_decap_resource_release(dev,
10879                                 dev_handle->dvh.rix_encap_decap);
10880                 if (dev_handle->dvh.modify_hdr)
10881                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
10882                 if (dev_handle->dvh.rix_push_vlan)
10883                         flow_dv_push_vlan_action_resource_release(dev,
10884                                                                   dev_handle);
10885                 if (dev_handle->dvh.rix_tag)
10886                         flow_dv_tag_release(dev,
10887                                             dev_handle->dvh.rix_tag);
10888                 flow_dv_fate_resource_release(dev, dev_handle);
10889                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10890                            tmp_idx);
10891         }
10892 }
10893
10894 /**
10895  * Query a dv flow  rule for its statistics via devx.
10896  *
10897  * @param[in] dev
10898  *   Pointer to Ethernet device.
10899  * @param[in] flow
10900  *   Pointer to the sub flow.
10901  * @param[out] data
10902  *   data retrieved by the query.
10903  * @param[out] error
10904  *   Perform verbose error reporting if not NULL.
10905  *
10906  * @return
10907  *   0 on success, a negative errno value otherwise and rte_errno is set.
10908  */
10909 static int
10910 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
10911                     void *data, struct rte_flow_error *error)
10912 {
10913         struct mlx5_priv *priv = dev->data->dev_private;
10914         struct rte_flow_query_count *qc = data;
10915
10916         if (!priv->config.devx)
10917                 return rte_flow_error_set(error, ENOTSUP,
10918                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10919                                           NULL,
10920                                           "counters are not supported");
10921         if (flow->counter) {
10922                 uint64_t pkts, bytes;
10923                 struct mlx5_flow_counter *cnt;
10924
10925                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
10926                                                  NULL);
10927                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
10928                                                &bytes);
10929
10930                 if (err)
10931                         return rte_flow_error_set(error, -err,
10932                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10933                                         NULL, "cannot read counters");
10934                 qc->hits_set = 1;
10935                 qc->bytes_set = 1;
10936                 qc->hits = pkts - cnt->hits;
10937                 qc->bytes = bytes - cnt->bytes;
10938                 if (qc->reset) {
10939                         cnt->hits = pkts;
10940                         cnt->bytes = bytes;
10941                 }
10942                 return 0;
10943         }
10944         return rte_flow_error_set(error, EINVAL,
10945                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10946                                   NULL,
10947                                   "counters are not available");
10948 }
10949
10950 /**
10951  * Query a flow rule AGE action for aging information.
10952  *
10953  * @param[in] dev
10954  *   Pointer to Ethernet device.
10955  * @param[in] flow
10956  *   Pointer to the sub flow.
10957  * @param[out] data
10958  *   data retrieved by the query.
10959  * @param[out] error
10960  *   Perform verbose error reporting if not NULL.
10961  *
10962  * @return
10963  *   0 on success, a negative errno value otherwise and rte_errno is set.
10964  */
10965 static int
10966 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
10967                   void *data, struct rte_flow_error *error)
10968 {
10969         struct rte_flow_query_age *resp = data;
10970
10971         if (flow->counter) {
10972                 struct mlx5_age_param *age_param =
10973                                 flow_dv_counter_idx_get_age(dev, flow->counter);
10974
10975                 if (!age_param || !age_param->timeout)
10976                         return rte_flow_error_set
10977                                         (error, EINVAL,
10978                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10979                                          NULL, "cannot read age data");
10980                 resp->aged = __atomic_load_n(&age_param->state,
10981                                              __ATOMIC_RELAXED) ==
10982                                                         AGE_TMOUT ? 1 : 0;
10983                 resp->sec_since_last_hit_valid = !resp->aged;
10984                 if (resp->sec_since_last_hit_valid)
10985                         resp->sec_since_last_hit =
10986                                 __atomic_load_n(&age_param->sec_since_last_hit,
10987                                                 __ATOMIC_RELAXED);
10988                 return 0;
10989         }
10990         return rte_flow_error_set(error, EINVAL,
10991                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10992                                   NULL,
10993                                   "age data not available");
10994 }
10995
10996 /**
10997  * Query a flow.
10998  *
10999  * @see rte_flow_query()
11000  * @see rte_flow_ops
11001  */
11002 static int
11003 flow_dv_query(struct rte_eth_dev *dev,
11004               struct rte_flow *flow __rte_unused,
11005               const struct rte_flow_action *actions __rte_unused,
11006               void *data __rte_unused,
11007               struct rte_flow_error *error __rte_unused)
11008 {
11009         int ret = -EINVAL;
11010
11011         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
11012                 switch (actions->type) {
11013                 case RTE_FLOW_ACTION_TYPE_VOID:
11014                         break;
11015                 case RTE_FLOW_ACTION_TYPE_COUNT:
11016                         ret = flow_dv_query_count(dev, flow, data, error);
11017                         break;
11018                 case RTE_FLOW_ACTION_TYPE_AGE:
11019                         ret = flow_dv_query_age(dev, flow, data, error);
11020                         break;
11021                 default:
11022                         return rte_flow_error_set(error, ENOTSUP,
11023                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11024                                                   actions,
11025                                                   "action not supported");
11026                 }
11027         }
11028         return ret;
11029 }
11030
11031 /**
11032  * Destroy the meter table set.
11033  * Lock free, (mutex should be acquired by caller).
11034  *
11035  * @param[in] dev
11036  *   Pointer to Ethernet device.
11037  * @param[in] tbl
11038  *   Pointer to the meter table set.
11039  *
11040  * @return
11041  *   Always 0.
11042  */
11043 static int
11044 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
11045                         struct mlx5_meter_domains_infos *tbl)
11046 {
11047         struct mlx5_priv *priv = dev->data->dev_private;
11048         struct mlx5_meter_domains_infos *mtd =
11049                                 (struct mlx5_meter_domains_infos *)tbl;
11050
11051         if (!mtd || !priv->config.dv_flow_en)
11052                 return 0;
11053         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
11054                 claim_zero(mlx5_flow_os_destroy_flow
11055                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
11056         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
11057                 claim_zero(mlx5_flow_os_destroy_flow
11058                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
11059         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
11060                 claim_zero(mlx5_flow_os_destroy_flow
11061                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
11062         if (mtd->egress.color_matcher)
11063                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11064                            (mtd->egress.color_matcher));
11065         if (mtd->egress.any_matcher)
11066                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11067                            (mtd->egress.any_matcher));
11068         if (mtd->egress.tbl)
11069                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
11070         if (mtd->egress.sfx_tbl)
11071                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
11072         if (mtd->ingress.color_matcher)
11073                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11074                            (mtd->ingress.color_matcher));
11075         if (mtd->ingress.any_matcher)
11076                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11077                            (mtd->ingress.any_matcher));
11078         if (mtd->ingress.tbl)
11079                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
11080         if (mtd->ingress.sfx_tbl)
11081                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
11082         if (mtd->transfer.color_matcher)
11083                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11084                            (mtd->transfer.color_matcher));
11085         if (mtd->transfer.any_matcher)
11086                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11087                            (mtd->transfer.any_matcher));
11088         if (mtd->transfer.tbl)
11089                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
11090         if (mtd->transfer.sfx_tbl)
11091                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
11092         if (mtd->drop_actn)
11093                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
11094         mlx5_free(mtd);
11095         return 0;
11096 }
11097
11098 /* Number of meter flow actions, count and jump or count and drop. */
11099 #define METER_ACTIONS 2
11100
11101 /**
11102  * Create specify domain meter table and suffix table.
11103  *
11104  * @param[in] dev
11105  *   Pointer to Ethernet device.
11106  * @param[in,out] mtb
11107  *   Pointer to DV meter table set.
11108  * @param[in] egress
11109  *   Table attribute.
11110  * @param[in] transfer
11111  *   Table attribute.
11112  * @param[in] color_reg_c_idx
11113  *   Reg C index for color match.
11114  *
11115  * @return
11116  *   0 on success, -1 otherwise and rte_errno is set.
11117  */
11118 static int
11119 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
11120                            struct mlx5_meter_domains_infos *mtb,
11121                            uint8_t egress, uint8_t transfer,
11122                            uint32_t color_reg_c_idx)
11123 {
11124         struct mlx5_priv *priv = dev->data->dev_private;
11125         struct mlx5_dev_ctx_shared *sh = priv->sh;
11126         struct mlx5_flow_dv_match_params mask = {
11127                 .size = sizeof(mask.buf),
11128         };
11129         struct mlx5_flow_dv_match_params value = {
11130                 .size = sizeof(value.buf),
11131         };
11132         struct mlx5dv_flow_matcher_attr dv_attr = {
11133                 .type = IBV_FLOW_ATTR_NORMAL,
11134                 .priority = 0,
11135                 .match_criteria_enable = 0,
11136                 .match_mask = (void *)&mask,
11137         };
11138         void *actions[METER_ACTIONS];
11139         struct mlx5_meter_domain_info *dtb;
11140         struct rte_flow_error error;
11141         int i = 0;
11142         int ret;
11143
11144         if (transfer)
11145                 dtb = &mtb->transfer;
11146         else if (egress)
11147                 dtb = &mtb->egress;
11148         else
11149                 dtb = &mtb->ingress;
11150         /* Create the meter table with METER level. */
11151         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
11152                                             egress, transfer, &error);
11153         if (!dtb->tbl) {
11154                 DRV_LOG(ERR, "Failed to create meter policer table.");
11155                 return -1;
11156         }
11157         /* Create the meter suffix table with SUFFIX level. */
11158         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
11159                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
11160                                             egress, transfer, &error);
11161         if (!dtb->sfx_tbl) {
11162                 DRV_LOG(ERR, "Failed to create meter suffix table.");
11163                 return -1;
11164         }
11165         /* Create matchers, Any and Color. */
11166         dv_attr.priority = 3;
11167         dv_attr.match_criteria_enable = 0;
11168         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11169                                                &dtb->any_matcher);
11170         if (ret) {
11171                 DRV_LOG(ERR, "Failed to create meter"
11172                              " policer default matcher.");
11173                 goto error_exit;
11174         }
11175         dv_attr.priority = 0;
11176         dv_attr.match_criteria_enable =
11177                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
11178         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
11179                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
11180         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11181                                                &dtb->color_matcher);
11182         if (ret) {
11183                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
11184                 goto error_exit;
11185         }
11186         if (mtb->count_actns[RTE_MTR_DROPPED])
11187                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
11188         actions[i++] = mtb->drop_actn;
11189         /* Default rule: lowest priority, match any, actions: drop. */
11190         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
11191                                        actions,
11192                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
11193         if (ret) {
11194                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
11195                 goto error_exit;
11196         }
11197         return 0;
11198 error_exit:
11199         return -1;
11200 }
11201
11202 /**
11203  * Create the needed meter and suffix tables.
11204  * Lock free, (mutex should be acquired by caller).
11205  *
11206  * @param[in] dev
11207  *   Pointer to Ethernet device.
11208  * @param[in] fm
11209  *   Pointer to the flow meter.
11210  *
11211  * @return
11212  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
11213  */
11214 static struct mlx5_meter_domains_infos *
11215 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
11216                        const struct mlx5_flow_meter *fm)
11217 {
11218         struct mlx5_priv *priv = dev->data->dev_private;
11219         struct mlx5_meter_domains_infos *mtb;
11220         int ret;
11221         int i;
11222
11223         if (!priv->mtr_en) {
11224                 rte_errno = ENOTSUP;
11225                 return NULL;
11226         }
11227         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
11228         if (!mtb) {
11229                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
11230                 return NULL;
11231         }
11232         /* Create meter count actions */
11233         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
11234                 struct mlx5_flow_counter *cnt;
11235                 if (!fm->policer_stats.cnt[i])
11236                         continue;
11237                 cnt = flow_dv_counter_get_by_idx(dev,
11238                       fm->policer_stats.cnt[i], NULL);
11239                 mtb->count_actns[i] = cnt->action;
11240         }
11241         /* Create drop action. */
11242         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
11243         if (ret) {
11244                 DRV_LOG(ERR, "Failed to create drop action.");
11245                 goto error_exit;
11246         }
11247         /* Egress meter table. */
11248         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
11249         if (ret) {
11250                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
11251                 goto error_exit;
11252         }
11253         /* Ingress meter table. */
11254         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
11255         if (ret) {
11256                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
11257                 goto error_exit;
11258         }
11259         /* FDB meter table. */
11260         if (priv->config.dv_esw_en) {
11261                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
11262                                                  priv->mtr_color_reg);
11263                 if (ret) {
11264                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
11265                         goto error_exit;
11266                 }
11267         }
11268         return mtb;
11269 error_exit:
11270         flow_dv_destroy_mtr_tbl(dev, mtb);
11271         return NULL;
11272 }
11273
11274 /**
11275  * Destroy domain policer rule.
11276  *
11277  * @param[in] dt
11278  *   Pointer to domain table.
11279  */
11280 static void
11281 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
11282 {
11283         int i;
11284
11285         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11286                 if (dt->policer_rules[i]) {
11287                         claim_zero(mlx5_flow_os_destroy_flow
11288                                    (dt->policer_rules[i]));
11289                         dt->policer_rules[i] = NULL;
11290                 }
11291         }
11292         if (dt->jump_actn) {
11293                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
11294                 dt->jump_actn = NULL;
11295         }
11296 }
11297
11298 /**
11299  * Destroy policer rules.
11300  *
11301  * @param[in] dev
11302  *   Pointer to Ethernet device.
11303  * @param[in] fm
11304  *   Pointer to flow meter structure.
11305  * @param[in] attr
11306  *   Pointer to flow attributes.
11307  *
11308  * @return
11309  *   Always 0.
11310  */
11311 static int
11312 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
11313                               const struct mlx5_flow_meter *fm,
11314                               const struct rte_flow_attr *attr)
11315 {
11316         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
11317
11318         if (!mtb)
11319                 return 0;
11320         if (attr->egress)
11321                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
11322         if (attr->ingress)
11323                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
11324         if (attr->transfer)
11325                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
11326         return 0;
11327 }
11328
11329 /**
11330  * Create specify domain meter policer rule.
11331  *
11332  * @param[in] fm
11333  *   Pointer to flow meter structure.
11334  * @param[in] mtb
11335  *   Pointer to DV meter table set.
11336  * @param[in] mtr_reg_c
11337  *   Color match REG_C.
11338  *
11339  * @return
11340  *   0 on success, -1 otherwise.
11341  */
11342 static int
11343 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
11344                                     struct mlx5_meter_domain_info *dtb,
11345                                     uint8_t mtr_reg_c)
11346 {
11347         struct mlx5_flow_dv_match_params matcher = {
11348                 .size = sizeof(matcher.buf),
11349         };
11350         struct mlx5_flow_dv_match_params value = {
11351                 .size = sizeof(value.buf),
11352         };
11353         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11354         void *actions[METER_ACTIONS];
11355         int i;
11356         int ret = 0;
11357
11358         /* Create jump action. */
11359         if (!dtb->jump_actn)
11360                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11361                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
11362         if (ret) {
11363                 DRV_LOG(ERR, "Failed to create policer jump action.");
11364                 goto error;
11365         }
11366         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11367                 int j = 0;
11368
11369                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
11370                                        rte_col_2_mlx5_col(i), UINT8_MAX);
11371                 if (mtb->count_actns[i])
11372                         actions[j++] = mtb->count_actns[i];
11373                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
11374                         actions[j++] = mtb->drop_actn;
11375                 else
11376                         actions[j++] = dtb->jump_actn;
11377                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
11378                                                (void *)&value, j, actions,
11379                                                &dtb->policer_rules[i]);
11380                 if (ret) {
11381                         DRV_LOG(ERR, "Failed to create policer rule.");
11382                         goto error;
11383                 }
11384         }
11385         return 0;
11386 error:
11387         rte_errno = errno;
11388         return -1;
11389 }
11390
11391 /**
11392  * Create policer rules.
11393  *
11394  * @param[in] dev
11395  *   Pointer to Ethernet device.
11396  * @param[in] fm
11397  *   Pointer to flow meter structure.
11398  * @param[in] attr
11399  *   Pointer to flow attributes.
11400  *
11401  * @return
11402  *   0 on success, -1 otherwise.
11403  */
11404 static int
11405 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
11406                              struct mlx5_flow_meter *fm,
11407                              const struct rte_flow_attr *attr)
11408 {
11409         struct mlx5_priv *priv = dev->data->dev_private;
11410         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11411         int ret;
11412
11413         if (attr->egress) {
11414                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
11415                                                 priv->mtr_color_reg);
11416                 if (ret) {
11417                         DRV_LOG(ERR, "Failed to create egress policer.");
11418                         goto error;
11419                 }
11420         }
11421         if (attr->ingress) {
11422                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
11423                                                 priv->mtr_color_reg);
11424                 if (ret) {
11425                         DRV_LOG(ERR, "Failed to create ingress policer.");
11426                         goto error;
11427                 }
11428         }
11429         if (attr->transfer) {
11430                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
11431                                                 priv->mtr_color_reg);
11432                 if (ret) {
11433                         DRV_LOG(ERR, "Failed to create transfer policer.");
11434                         goto error;
11435                 }
11436         }
11437         return 0;
11438 error:
11439         flow_dv_destroy_policer_rules(dev, fm, attr);
11440         return -1;
11441 }
11442
11443 /**
11444  * Query a devx counter.
11445  *
11446  * @param[in] dev
11447  *   Pointer to the Ethernet device structure.
11448  * @param[in] cnt
11449  *   Index to the flow counter.
11450  * @param[in] clear
11451  *   Set to clear the counter statistics.
11452  * @param[out] pkts
11453  *   The statistics value of packets.
11454  * @param[out] bytes
11455  *   The statistics value of bytes.
11456  *
11457  * @return
11458  *   0 on success, otherwise return -1.
11459  */
11460 static int
11461 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
11462                       uint64_t *pkts, uint64_t *bytes)
11463 {
11464         struct mlx5_priv *priv = dev->data->dev_private;
11465         struct mlx5_flow_counter *cnt;
11466         uint64_t inn_pkts, inn_bytes;
11467         int ret;
11468
11469         if (!priv->config.devx)
11470                 return -1;
11471
11472         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
11473         if (ret)
11474                 return -1;
11475         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
11476         *pkts = inn_pkts - cnt->hits;
11477         *bytes = inn_bytes - cnt->bytes;
11478         if (clear) {
11479                 cnt->hits = inn_pkts;
11480                 cnt->bytes = inn_bytes;
11481         }
11482         return 0;
11483 }
11484
11485 /**
11486  * Get aged-out flows.
11487  *
11488  * @param[in] dev
11489  *   Pointer to the Ethernet device structure.
11490  * @param[in] context
11491  *   The address of an array of pointers to the aged-out flows contexts.
11492  * @param[in] nb_contexts
11493  *   The length of context array pointers.
11494  * @param[out] error
11495  *   Perform verbose error reporting if not NULL. Initialized in case of
11496  *   error only.
11497  *
11498  * @return
11499  *   how many contexts get in success, otherwise negative errno value.
11500  *   if nb_contexts is 0, return the amount of all aged contexts.
11501  *   if nb_contexts is not 0 , return the amount of aged flows reported
11502  *   in the context array.
11503  * @note: only stub for now
11504  */
11505 static int
11506 flow_get_aged_flows(struct rte_eth_dev *dev,
11507                     void **context,
11508                     uint32_t nb_contexts,
11509                     struct rte_flow_error *error)
11510 {
11511         struct mlx5_priv *priv = dev->data->dev_private;
11512         struct mlx5_age_info *age_info;
11513         struct mlx5_age_param *age_param;
11514         struct mlx5_flow_counter *counter;
11515         int nb_flows = 0;
11516
11517         if (nb_contexts && !context)
11518                 return rte_flow_error_set(error, EINVAL,
11519                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11520                                           NULL,
11521                                           "Should assign at least one flow or"
11522                                           " context to get if nb_contexts != 0");
11523         age_info = GET_PORT_AGE_INFO(priv);
11524         rte_spinlock_lock(&age_info->aged_sl);
11525         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
11526                 nb_flows++;
11527                 if (nb_contexts) {
11528                         age_param = MLX5_CNT_TO_AGE(counter);
11529                         context[nb_flows - 1] = age_param->context;
11530                         if (!(--nb_contexts))
11531                                 break;
11532                 }
11533         }
11534         rte_spinlock_unlock(&age_info->aged_sl);
11535         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
11536         return nb_flows;
11537 }
11538
11539 /*
11540  * Mutex-protected thunk to lock-free  __flow_dv_translate().
11541  */
11542 static int
11543 flow_dv_translate(struct rte_eth_dev *dev,
11544                   struct mlx5_flow *dev_flow,
11545                   const struct rte_flow_attr *attr,
11546                   const struct rte_flow_item items[],
11547                   const struct rte_flow_action actions[],
11548                   struct rte_flow_error *error)
11549 {
11550         int ret;
11551
11552         flow_dv_shared_lock(dev);
11553         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
11554         flow_dv_shared_unlock(dev);
11555         return ret;
11556 }
11557
11558 /*
11559  * Mutex-protected thunk to lock-free  __flow_dv_apply().
11560  */
11561 static int
11562 flow_dv_apply(struct rte_eth_dev *dev,
11563               struct rte_flow *flow,
11564               struct rte_flow_error *error)
11565 {
11566         int ret;
11567
11568         flow_dv_shared_lock(dev);
11569         ret = __flow_dv_apply(dev, flow, error);
11570         flow_dv_shared_unlock(dev);
11571         return ret;
11572 }
11573
11574 /*
11575  * Mutex-protected thunk to lock-free __flow_dv_remove().
11576  */
11577 static void
11578 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
11579 {
11580         flow_dv_shared_lock(dev);
11581         __flow_dv_remove(dev, flow);
11582         flow_dv_shared_unlock(dev);
11583 }
11584
11585 /*
11586  * Mutex-protected thunk to lock-free __flow_dv_destroy().
11587  */
11588 static void
11589 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
11590 {
11591         flow_dv_shared_lock(dev);
11592         __flow_dv_destroy(dev, flow);
11593         flow_dv_shared_unlock(dev);
11594 }
11595
11596 /*
11597  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
11598  */
11599 static uint32_t
11600 flow_dv_counter_allocate(struct rte_eth_dev *dev)
11601 {
11602         uint32_t cnt;
11603
11604         flow_dv_shared_lock(dev);
11605         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
11606         flow_dv_shared_unlock(dev);
11607         return cnt;
11608 }
11609
11610 /*
11611  * Mutex-protected thunk to lock-free flow_dv_counter_release().
11612  */
11613 static void
11614 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
11615 {
11616         flow_dv_shared_lock(dev);
11617         flow_dv_counter_release(dev, cnt);
11618         flow_dv_shared_unlock(dev);
11619 }
11620
11621 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
11622         .validate = flow_dv_validate,
11623         .prepare = flow_dv_prepare,
11624         .translate = flow_dv_translate,
11625         .apply = flow_dv_apply,
11626         .remove = flow_dv_remove,
11627         .destroy = flow_dv_destroy,
11628         .query = flow_dv_query,
11629         .create_mtr_tbls = flow_dv_create_mtr_tbl,
11630         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
11631         .create_policer_rules = flow_dv_create_policer_rules,
11632         .destroy_policer_rules = flow_dv_destroy_policer_rules,
11633         .counter_alloc = flow_dv_counter_allocate,
11634         .counter_free = flow_dv_counter_free,
11635         .counter_query = flow_dv_counter_query,
11636         .get_aged_flows = flow_get_aged_flows,
11637 };
11638
11639 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
11640