drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24 #include <rte_mtr.h>
  25 #include <rte_mtr_driver.h>
  26
  27 #include <mlx5_glue.h>
  28 #include <mlx5_devx_cmds.h>
  29 #include <mlx5_prm.h>
  30 #include <mlx5_malloc.h>
  31
  32 #include "mlx5_defs.h"
  33 #include "mlx5.h"
  34 #include "mlx5_common_os.h"
  35 #include "mlx5_flow.h"
  36 #include "mlx5_flow_os.h"
  37 #include "mlx5_rx.h"
  38 #include "mlx5_tx.h"
  39 #include "rte_pmd_mlx5.h"
  40
  41 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  42
  43 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  44 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  45 #endif
  46
  47 #ifndef HAVE_MLX5DV_DR_ESWITCH
  48 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  49 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  50 #endif
  51 #endif
  52
  53 #ifndef HAVE_MLX5DV_DR
  54 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  55 #endif
  56
  57 /* VLAN header definitions */
  58 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  59 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  60 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  61 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  62 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  63
  64 union flow_dv_attr {
  65         struct {
  66                 uint32_t valid:1;
  67                 uint32_t ipv4:1;
  68                 uint32_t ipv6:1;
  69                 uint32_t tcp:1;
  70                 uint32_t udp:1;
  71                 uint32_t reserved:27;
  72         };
  73         uint32_t attr;
  74 };
  75
  76 static int
  77 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
  78                              struct mlx5_flow_tbl_resource *tbl);
  79
  80 static int
  81 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  82                                      uint32_t encap_decap_idx);
  83
  84 static int
  85 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  86                                         uint32_t port_id);
  87 static void
  88 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);
  89
  90 static int
  91 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
  92                                   uint32_t rix_jump);
  93
  94 /**
  95  * Initialize flow attributes structure according to flow items' types.
  96  *
  97  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  98  * mode. For tunnel mode, the items to be modified are the outermost ones.
  99  *
 100  * @param[in] item
 101  *   Pointer to item specification.
 102  * @param[out] attr
 103  *   Pointer to flow attributes structure.
 104  * @param[in] dev_flow
 105  *   Pointer to the sub flow.
 106  * @param[in] tunnel_decap
 107  *   Whether action is after tunnel decapsulation.
 108  */
 109 static void
 110 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 111                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 112 {
 113         uint64_t layers = dev_flow->handle->layers;
 114
 115         /*
 116          * If layers is already initialized, it means this dev_flow is the
 117          * suffix flow, the layers flags is set by the prefix flow. Need to
 118          * use the layer flags from prefix flow as the suffix flow may not
 119          * have the user defined items as the flow is split.
 120          */
 121         if (layers) {
 122                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 123                         attr->ipv4 = 1;
 124                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 125                         attr->ipv6 = 1;
 126                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 127                         attr->tcp = 1;
 128                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 129                         attr->udp = 1;
 130                 attr->valid = 1;
 131                 return;
 132         }
 133         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 134                 uint8_t next_protocol = 0xff;
 135                 switch (item->type) {
 136                 case RTE_FLOW_ITEM_TYPE_GRE:
 137                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 138                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 139                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 140                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 141                 case RTE_FLOW_ITEM_TYPE_MPLS:
 142                         if (tunnel_decap)
 143                                 attr->attr = 0;
 144                         break;
 145                 case RTE_FLOW_ITEM_TYPE_IPV4:
 146                         if (!attr->ipv6)
 147                                 attr->ipv4 = 1;
 148                         if (item->mask != NULL &&
 149                             ((const struct rte_flow_item_ipv4 *)
 150                             item->mask)->hdr.next_proto_id)
 151                                 next_protocol =
 152                                     ((const struct rte_flow_item_ipv4 *)
 153                                       (item->spec))->hdr.next_proto_id &
 154                                     ((const struct rte_flow_item_ipv4 *)
 155                                       (item->mask))->hdr.next_proto_id;
 156                         if ((next_protocol == IPPROTO_IPIP ||
 157                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 158                                 attr->attr = 0;
 159                         break;
 160                 case RTE_FLOW_ITEM_TYPE_IPV6:
 161                         if (!attr->ipv4)
 162                                 attr->ipv6 = 1;
 163                         if (item->mask != NULL &&
 164                             ((const struct rte_flow_item_ipv6 *)
 165                             item->mask)->hdr.proto)
 166                                 next_protocol =
 167                                     ((const struct rte_flow_item_ipv6 *)
 168                                       (item->spec))->hdr.proto &
 169                                     ((const struct rte_flow_item_ipv6 *)
 170                                       (item->mask))->hdr.proto;
 171                         if ((next_protocol == IPPROTO_IPIP ||
 172                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 173                                 attr->attr = 0;
 174                         break;
 175                 case RTE_FLOW_ITEM_TYPE_UDP:
 176                         if (!attr->tcp)
 177                                 attr->udp = 1;
 178                         break;
 179                 case RTE_FLOW_ITEM_TYPE_TCP:
 180                         if (!attr->udp)
 181                                 attr->tcp = 1;
 182                         break;
 183                 default:
 184                         break;
 185                 }
 186         }
 187         attr->valid = 1;
 188 }
 189
 190 /**
 191  * Convert rte_mtr_color to mlx5 color.
 192  *
 193  * @param[in] rcol
 194  *   rte_mtr_color.
 195  *
 196  * @return
 197  *   mlx5 color.
 198  */
 199 static int
 200 rte_col_2_mlx5_col(enum rte_color rcol)
 201 {
 202         switch (rcol) {
 203         case RTE_COLOR_GREEN:
 204                 return MLX5_FLOW_COLOR_GREEN;
 205         case RTE_COLOR_YELLOW:
 206                 return MLX5_FLOW_COLOR_YELLOW;
 207         case RTE_COLOR_RED:
 208                 return MLX5_FLOW_COLOR_RED;
 209         default:
 210                 break;
 211         }
 212         return MLX5_FLOW_COLOR_UNDEFINED;
 213 }
 214
 215 struct field_modify_info {
 216         uint32_t size; /* Size of field in protocol header, in bytes. */
 217         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 218         enum mlx5_modification_field id;
 219 };
 220
 221 struct field_modify_info modify_eth[] = {
 222         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 223         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 224         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 225         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 226         {0, 0, 0},
 227 };
 228
 229 struct field_modify_info modify_vlan_out_first_vid[] = {
 230         /* Size in bits !!! */
 231         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 232         {0, 0, 0},
 233 };
 234
 235 struct field_modify_info modify_ipv4[] = {
 236         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 237         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 238         {4, 12, MLX5_MODI_OUT_SIPV4},
 239         {4, 16, MLX5_MODI_OUT_DIPV4},
 240         {0, 0, 0},
 241 };
 242
 243 struct field_modify_info modify_ipv6[] = {
 244         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 245         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 246         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 247         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 248         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 249         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 250         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 251         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 252         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 253         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 254         {0, 0, 0},
 255 };
 256
 257 struct field_modify_info modify_udp[] = {
 258         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 259         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 260         {0, 0, 0},
 261 };
 262
 263 struct field_modify_info modify_tcp[] = {
 264         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 265         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 266         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 267         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 268         {0, 0, 0},
 269 };
 270
 271 static const struct rte_flow_item *
 272 mlx5_flow_find_tunnel_item(const struct rte_flow_item *item)
 273 {
 274         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 275                 switch (item->type) {
 276                 default:
 277                         break;
 278                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 279                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 280                 case RTE_FLOW_ITEM_TYPE_GRE:
 281                 case RTE_FLOW_ITEM_TYPE_MPLS:
 282                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 283                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 284                         return item;
 285                 case RTE_FLOW_ITEM_TYPE_IPV4:
 286                 case RTE_FLOW_ITEM_TYPE_IPV6:
 287                         if (item[1].type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 288                             item[1].type == RTE_FLOW_ITEM_TYPE_IPV6)
 289                                 return item;
 290                         break;
 291                 }
 292         }
 293         return NULL;
 294 }
 295
 296 static void
 297 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 298                           uint8_t next_protocol, uint64_t *item_flags,
 299                           int *tunnel)
 300 {
 301         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 302                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 303         if (next_protocol == IPPROTO_IPIP) {
 304                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 305                 *tunnel = 1;
 306         }
 307         if (next_protocol == IPPROTO_IPV6) {
 308                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 309                 *tunnel = 1;
 310         }
 311 }
 312
 313 /* Update VLAN's VID/PCP based on input rte_flow_action.
 314  *
 315  * @param[in] action
 316  *   Pointer to struct rte_flow_action.
 317  * @param[out] vlan
 318  *   Pointer to struct rte_vlan_hdr.
 319  */
 320 static void
 321 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 322                          struct rte_vlan_hdr *vlan)
 323 {
 324         uint16_t vlan_tci;
 325         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 326                 vlan_tci =
 327                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 328                                                action->conf)->vlan_pcp;
 329                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 330                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 331                 vlan->vlan_tci |= vlan_tci;
 332         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 333                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 334                 vlan->vlan_tci |= rte_be_to_cpu_16
 335                     (((const struct rte_flow_action_of_set_vlan_vid *)
 336                                              action->conf)->vlan_vid);
 337         }
 338 }
 339
 340 /**
 341  * Fetch 1, 2, 3 or 4 byte field from the byte array
 342  * and return as unsigned integer in host-endian format.
 343  *
 344  * @param[in] data
 345  *   Pointer to data array.
 346  * @param[in] size
 347  *   Size of field to extract.
 348  *
 349  * @return
 350  *   converted field in host endian format.
 351  */
 352 static inline uint32_t
 353 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 354 {
 355         uint32_t ret;
 356
 357         switch (size) {
 358         case 1:
 359                 ret = *data;
 360                 break;
 361         case 2:
 362                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 363                 break;
 364         case 3:
 365                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 366                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 367                 break;
 368         case 4:
 369                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 370                 break;
 371         default:
 372                 MLX5_ASSERT(false);
 373                 ret = 0;
 374                 break;
 375         }
 376         return ret;
 377 }
 378
 379 /**
 380  * Convert modify-header action to DV specification.
 381  *
 382  * Data length of each action is determined by provided field description
 383  * and the item mask. Data bit offset and width of each action is determined
 384  * by provided item mask.
 385  *
 386  * @param[in] item
 387  *   Pointer to item specification.
 388  * @param[in] field
 389  *   Pointer to field modification information.
 390  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 391  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 392  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 393  * @param[in] dcopy
 394  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 395  *   Negative offset value sets the same offset as source offset.
 396  *   size field is ignored, value is taken from source field.
 397  * @param[in,out] resource
 398  *   Pointer to the modify-header resource.
 399  * @param[in] type
 400  *   Type of modification.
 401  * @param[out] error
 402  *   Pointer to the error structure.
 403  *
 404  * @return
 405  *   0 on success, a negative errno value otherwise and rte_errno is set.
 406  */
 407 static int
 408 flow_dv_convert_modify_action(struct rte_flow_item *item,
 409                               struct field_modify_info *field,
 410                               struct field_modify_info *dcopy,
 411                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 412                               uint32_t type, struct rte_flow_error *error)
 413 {
 414         uint32_t i = resource->actions_num;
 415         struct mlx5_modification_cmd *actions = resource->actions;
 416
 417         /*
 418          * The item and mask are provided in big-endian format.
 419          * The fields should be presented as in big-endian format either.
 420          * Mask must be always present, it defines the actual field width.
 421          */
 422         MLX5_ASSERT(item->mask);
 423         MLX5_ASSERT(field->size);
 424         do {
 425                 unsigned int size_b;
 426                 unsigned int off_b;
 427                 uint32_t mask;
 428                 uint32_t data;
 429
 430                 if (i >= MLX5_MAX_MODIFY_NUM)
 431                         return rte_flow_error_set(error, EINVAL,
 432                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 433                                  "too many items to modify");
 434                 /* Fetch variable byte size mask from the array. */
 435                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 436                                            field->offset, field->size);
 437                 if (!mask) {
 438                         ++field;
 439                         continue;
 440                 }
 441                 /* Deduce actual data width in bits from mask value. */
 442                 off_b = rte_bsf32(mask);
 443                 size_b = sizeof(uint32_t) * CHAR_BIT -
 444                          off_b - __builtin_clz(mask);
 445                 MLX5_ASSERT(size_b);
 446                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 447                 actions[i] = (struct mlx5_modification_cmd) {
 448                         .action_type = type,
 449                         .field = field->id,
 450                         .offset = off_b,
 451                         .length = size_b,
 452                 };
 453                 /* Convert entire record to expected big-endian format. */
 454                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 455                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 456                         MLX5_ASSERT(dcopy);
 457                         actions[i].dst_field = dcopy->id;
 458                         actions[i].dst_offset =
 459                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 460                         /* Convert entire record to big-endian format. */
 461                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 462                         ++dcopy;
 463                 } else {
 464                         MLX5_ASSERT(item->spec);
 465                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 466                                                    field->offset, field->size);
 467                         /* Shift out the trailing masked bits from data. */
 468                         data = (data & mask) >> off_b;
 469                         actions[i].data1 = rte_cpu_to_be_32(data);
 470                 }
 471                 ++i;
 472                 ++field;
 473         } while (field->size);
 474         if (resource->actions_num == i)
 475                 return rte_flow_error_set(error, EINVAL,
 476                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 477                                           "invalid modification flow item");
 478         resource->actions_num = i;
 479         return 0;
 480 }
 481
 482 /**
 483  * Convert modify-header set IPv4 address action to DV specification.
 484  *
 485  * @param[in,out] resource
 486  *   Pointer to the modify-header resource.
 487  * @param[in] action
 488  *   Pointer to action specification.
 489  * @param[out] error
 490  *   Pointer to the error structure.
 491  *
 492  * @return
 493  *   0 on success, a negative errno value otherwise and rte_errno is set.
 494  */
 495 static int
 496 flow_dv_convert_action_modify_ipv4
 497                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 498                          const struct rte_flow_action *action,
 499                          struct rte_flow_error *error)
 500 {
 501         const struct rte_flow_action_set_ipv4 *conf =
 502                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 503         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 504         struct rte_flow_item_ipv4 ipv4;
 505         struct rte_flow_item_ipv4 ipv4_mask;
 506
 507         memset(&ipv4, 0, sizeof(ipv4));
 508         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 509         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 510                 ipv4.hdr.src_addr = conf->ipv4_addr;
 511                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 512         } else {
 513                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 514                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 515         }
 516         item.spec = &ipv4;
 517         item.mask = &ipv4_mask;
 518         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 519                                              MLX5_MODIFICATION_TYPE_SET, error);
 520 }
 521
 522 /**
 523  * Convert modify-header set IPv6 address action to DV specification.
 524  *
 525  * @param[in,out] resource
 526  *   Pointer to the modify-header resource.
 527  * @param[in] action
 528  *   Pointer to action specification.
 529  * @param[out] error
 530  *   Pointer to the error structure.
 531  *
 532  * @return
 533  *   0 on success, a negative errno value otherwise and rte_errno is set.
 534  */
 535 static int
 536 flow_dv_convert_action_modify_ipv6
 537                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 538                          const struct rte_flow_action *action,
 539                          struct rte_flow_error *error)
 540 {
 541         const struct rte_flow_action_set_ipv6 *conf =
 542                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 543         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 544         struct rte_flow_item_ipv6 ipv6;
 545         struct rte_flow_item_ipv6 ipv6_mask;
 546
 547         memset(&ipv6, 0, sizeof(ipv6));
 548         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 549         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 550                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 551                        sizeof(ipv6.hdr.src_addr));
 552                 memcpy(&ipv6_mask.hdr.src_addr,
 553                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 554                        sizeof(ipv6.hdr.src_addr));
 555         } else {
 556                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.dst_addr));
 558                 memcpy(&ipv6_mask.hdr.dst_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 560                        sizeof(ipv6.hdr.dst_addr));
 561         }
 562         item.spec = &ipv6;
 563         item.mask = &ipv6_mask;
 564         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 565                                              MLX5_MODIFICATION_TYPE_SET, error);
 566 }
 567
 568 /**
 569  * Convert modify-header set MAC address action to DV specification.
 570  *
 571  * @param[in,out] resource
 572  *   Pointer to the modify-header resource.
 573  * @param[in] action
 574  *   Pointer to action specification.
 575  * @param[out] error
 576  *   Pointer to the error structure.
 577  *
 578  * @return
 579  *   0 on success, a negative errno value otherwise and rte_errno is set.
 580  */
 581 static int
 582 flow_dv_convert_action_modify_mac
 583                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 584                          const struct rte_flow_action *action,
 585                          struct rte_flow_error *error)
 586 {
 587         const struct rte_flow_action_set_mac *conf =
 588                 (const struct rte_flow_action_set_mac *)(action->conf);
 589         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 590         struct rte_flow_item_eth eth;
 591         struct rte_flow_item_eth eth_mask;
 592
 593         memset(&eth, 0, sizeof(eth));
 594         memset(&eth_mask, 0, sizeof(eth_mask));
 595         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 596                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 597                        sizeof(eth.src.addr_bytes));
 598                 memcpy(&eth_mask.src.addr_bytes,
 599                        &rte_flow_item_eth_mask.src.addr_bytes,
 600                        sizeof(eth_mask.src.addr_bytes));
 601         } else {
 602                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.dst.addr_bytes));
 604                 memcpy(&eth_mask.dst.addr_bytes,
 605                        &rte_flow_item_eth_mask.dst.addr_bytes,
 606                        sizeof(eth_mask.dst.addr_bytes));
 607         }
 608         item.spec = &eth;
 609         item.mask = &eth_mask;
 610         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 611                                              MLX5_MODIFICATION_TYPE_SET, error);
 612 }
 613
 614 /**
 615  * Convert modify-header set VLAN VID action to DV specification.
 616  *
 617  * @param[in,out] resource
 618  *   Pointer to the modify-header resource.
 619  * @param[in] action
 620  *   Pointer to action specification.
 621  * @param[out] error
 622  *   Pointer to the error structure.
 623  *
 624  * @return
 625  *   0 on success, a negative errno value otherwise and rte_errno is set.
 626  */
 627 static int
 628 flow_dv_convert_action_modify_vlan_vid
 629                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 630                          const struct rte_flow_action *action,
 631                          struct rte_flow_error *error)
 632 {
 633         const struct rte_flow_action_of_set_vlan_vid *conf =
 634                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 635         int i = resource->actions_num;
 636         struct mlx5_modification_cmd *actions = resource->actions;
 637         struct field_modify_info *field = modify_vlan_out_first_vid;
 638
 639         if (i >= MLX5_MAX_MODIFY_NUM)
 640                 return rte_flow_error_set(error, EINVAL,
 641                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 642                          "too many items to modify");
 643         actions[i] = (struct mlx5_modification_cmd) {
 644                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 645                 .field = field->id,
 646                 .length = field->size,
 647                 .offset = field->offset,
 648         };
 649         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 650         actions[i].data1 = conf->vlan_vid;
 651         actions[i].data1 = actions[i].data1 << 16;
 652         resource->actions_num = ++i;
 653         return 0;
 654 }
 655
 656 /**
 657  * Convert modify-header set TP action to DV specification.
 658  *
 659  * @param[in,out] resource
 660  *   Pointer to the modify-header resource.
 661  * @param[in] action
 662  *   Pointer to action specification.
 663  * @param[in] items
 664  *   Pointer to rte_flow_item objects list.
 665  * @param[in] attr
 666  *   Pointer to flow attributes structure.
 667  * @param[in] dev_flow
 668  *   Pointer to the sub flow.
 669  * @param[in] tunnel_decap
 670  *   Whether action is after tunnel decapsulation.
 671  * @param[out] error
 672  *   Pointer to the error structure.
 673  *
 674  * @return
 675  *   0 on success, a negative errno value otherwise and rte_errno is set.
 676  */
 677 static int
 678 flow_dv_convert_action_modify_tp
 679                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 680                          const struct rte_flow_action *action,
 681                          const struct rte_flow_item *items,
 682                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 683                          bool tunnel_decap, struct rte_flow_error *error)
 684 {
 685         const struct rte_flow_action_set_tp *conf =
 686                 (const struct rte_flow_action_set_tp *)(action->conf);
 687         struct rte_flow_item item;
 688         struct rte_flow_item_udp udp;
 689         struct rte_flow_item_udp udp_mask;
 690         struct rte_flow_item_tcp tcp;
 691         struct rte_flow_item_tcp tcp_mask;
 692         struct field_modify_info *field;
 693
 694         if (!attr->valid)
 695                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 696         if (attr->udp) {
 697                 memset(&udp, 0, sizeof(udp));
 698                 memset(&udp_mask, 0, sizeof(udp_mask));
 699                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 700                         udp.hdr.src_port = conf->port;
 701                         udp_mask.hdr.src_port =
 702                                         rte_flow_item_udp_mask.hdr.src_port;
 703                 } else {
 704                         udp.hdr.dst_port = conf->port;
 705                         udp_mask.hdr.dst_port =
 706                                         rte_flow_item_udp_mask.hdr.dst_port;
 707                 }
 708                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 709                 item.spec = &udp;
 710                 item.mask = &udp_mask;
 711                 field = modify_udp;
 712         } else {
 713                 MLX5_ASSERT(attr->tcp);
 714                 memset(&tcp, 0, sizeof(tcp));
 715                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 716                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 717                         tcp.hdr.src_port = conf->port;
 718                         tcp_mask.hdr.src_port =
 719                                         rte_flow_item_tcp_mask.hdr.src_port;
 720                 } else {
 721                         tcp.hdr.dst_port = conf->port;
 722                         tcp_mask.hdr.dst_port =
 723                                         rte_flow_item_tcp_mask.hdr.dst_port;
 724                 }
 725                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 726                 item.spec = &tcp;
 727                 item.mask = &tcp_mask;
 728                 field = modify_tcp;
 729         }
 730         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 731                                              MLX5_MODIFICATION_TYPE_SET, error);
 732 }
 733
 734 /**
 735  * Convert modify-header set TTL action to DV specification.
 736  *
 737  * @param[in,out] resource
 738  *   Pointer to the modify-header resource.
 739  * @param[in] action
 740  *   Pointer to action specification.
 741  * @param[in] items
 742  *   Pointer to rte_flow_item objects list.
 743  * @param[in] attr
 744  *   Pointer to flow attributes structure.
 745  * @param[in] dev_flow
 746  *   Pointer to the sub flow.
 747  * @param[in] tunnel_decap
 748  *   Whether action is after tunnel decapsulation.
 749  * @param[out] error
 750  *   Pointer to the error structure.
 751  *
 752  * @return
 753  *   0 on success, a negative errno value otherwise and rte_errno is set.
 754  */
 755 static int
 756 flow_dv_convert_action_modify_ttl
 757                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 758                          const struct rte_flow_action *action,
 759                          const struct rte_flow_item *items,
 760                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 761                          bool tunnel_decap, struct rte_flow_error *error)
 762 {
 763         const struct rte_flow_action_set_ttl *conf =
 764                 (const struct rte_flow_action_set_ttl *)(action->conf);
 765         struct rte_flow_item item;
 766         struct rte_flow_item_ipv4 ipv4;
 767         struct rte_flow_item_ipv4 ipv4_mask;
 768         struct rte_flow_item_ipv6 ipv6;
 769         struct rte_flow_item_ipv6 ipv6_mask;
 770         struct field_modify_info *field;
 771
 772         if (!attr->valid)
 773                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 774         if (attr->ipv4) {
 775                 memset(&ipv4, 0, sizeof(ipv4));
 776                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 777                 ipv4.hdr.time_to_live = conf->ttl_value;
 778                 ipv4_mask.hdr.time_to_live = 0xFF;
 779                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 780                 item.spec = &ipv4;
 781                 item.mask = &ipv4_mask;
 782                 field = modify_ipv4;
 783         } else {
 784                 MLX5_ASSERT(attr->ipv6);
 785                 memset(&ipv6, 0, sizeof(ipv6));
 786                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 787                 ipv6.hdr.hop_limits = conf->ttl_value;
 788                 ipv6_mask.hdr.hop_limits = 0xFF;
 789                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 790                 item.spec = &ipv6;
 791                 item.mask = &ipv6_mask;
 792                 field = modify_ipv6;
 793         }
 794         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 795                                              MLX5_MODIFICATION_TYPE_SET, error);
 796 }
 797
 798 /**
 799  * Convert modify-header decrement TTL action to DV specification.
 800  *
 801  * @param[in,out] resource
 802  *   Pointer to the modify-header resource.
 803  * @param[in] action
 804  *   Pointer to action specification.
 805  * @param[in] items
 806  *   Pointer to rte_flow_item objects list.
 807  * @param[in] attr
 808  *   Pointer to flow attributes structure.
 809  * @param[in] dev_flow
 810  *   Pointer to the sub flow.
 811  * @param[in] tunnel_decap
 812  *   Whether action is after tunnel decapsulation.
 813  * @param[out] error
 814  *   Pointer to the error structure.
 815  *
 816  * @return
 817  *   0 on success, a negative errno value otherwise and rte_errno is set.
 818  */
 819 static int
 820 flow_dv_convert_action_modify_dec_ttl
 821                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 822                          const struct rte_flow_item *items,
 823                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 824                          bool tunnel_decap, struct rte_flow_error *error)
 825 {
 826         struct rte_flow_item item;
 827         struct rte_flow_item_ipv4 ipv4;
 828         struct rte_flow_item_ipv4 ipv4_mask;
 829         struct rte_flow_item_ipv6 ipv6;
 830         struct rte_flow_item_ipv6 ipv6_mask;
 831         struct field_modify_info *field;
 832
 833         if (!attr->valid)
 834                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 835         if (attr->ipv4) {
 836                 memset(&ipv4, 0, sizeof(ipv4));
 837                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 838                 ipv4.hdr.time_to_live = 0xFF;
 839                 ipv4_mask.hdr.time_to_live = 0xFF;
 840                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 841                 item.spec = &ipv4;
 842                 item.mask = &ipv4_mask;
 843                 field = modify_ipv4;
 844         } else {
 845                 MLX5_ASSERT(attr->ipv6);
 846                 memset(&ipv6, 0, sizeof(ipv6));
 847                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 848                 ipv6.hdr.hop_limits = 0xFF;
 849                 ipv6_mask.hdr.hop_limits = 0xFF;
 850                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 851                 item.spec = &ipv6;
 852                 item.mask = &ipv6_mask;
 853                 field = modify_ipv6;
 854         }
 855         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 856                                              MLX5_MODIFICATION_TYPE_ADD, error);
 857 }
 858
 859 /**
 860  * Convert modify-header increment/decrement TCP Sequence number
 861  * to DV specification.
 862  *
 863  * @param[in,out] resource
 864  *   Pointer to the modify-header resource.
 865  * @param[in] action
 866  *   Pointer to action specification.
 867  * @param[out] error
 868  *   Pointer to the error structure.
 869  *
 870  * @return
 871  *   0 on success, a negative errno value otherwise and rte_errno is set.
 872  */
 873 static int
 874 flow_dv_convert_action_modify_tcp_seq
 875                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 876                          const struct rte_flow_action *action,
 877                          struct rte_flow_error *error)
 878 {
 879         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 880         uint64_t value = rte_be_to_cpu_32(*conf);
 881         struct rte_flow_item item;
 882         struct rte_flow_item_tcp tcp;
 883         struct rte_flow_item_tcp tcp_mask;
 884
 885         memset(&tcp, 0, sizeof(tcp));
 886         memset(&tcp_mask, 0, sizeof(tcp_mask));
 887         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 888                 /*
 889                  * The HW has no decrement operation, only increment operation.
 890                  * To simulate decrement X from Y using increment operation
 891                  * we need to add UINT32_MAX X times to Y.
 892                  * Each adding of UINT32_MAX decrements Y by 1.
 893                  */
 894                 value *= UINT32_MAX;
 895         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 896         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 897         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 898         item.spec = &tcp;
 899         item.mask = &tcp_mask;
 900         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 901                                              MLX5_MODIFICATION_TYPE_ADD, error);
 902 }
 903
 904 /**
 905  * Convert modify-header increment/decrement TCP Acknowledgment number
 906  * to DV specification.
 907  *
 908  * @param[in,out] resource
 909  *   Pointer to the modify-header resource.
 910  * @param[in] action
 911  *   Pointer to action specification.
 912  * @param[out] error
 913  *   Pointer to the error structure.
 914  *
 915  * @return
 916  *   0 on success, a negative errno value otherwise and rte_errno is set.
 917  */
 918 static int
 919 flow_dv_convert_action_modify_tcp_ack
 920                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 921                          const struct rte_flow_action *action,
 922                          struct rte_flow_error *error)
 923 {
 924         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 925         uint64_t value = rte_be_to_cpu_32(*conf);
 926         struct rte_flow_item item;
 927         struct rte_flow_item_tcp tcp;
 928         struct rte_flow_item_tcp tcp_mask;
 929
 930         memset(&tcp, 0, sizeof(tcp));
 931         memset(&tcp_mask, 0, sizeof(tcp_mask));
 932         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 933                 /*
 934                  * The HW has no decrement operation, only increment operation.
 935                  * To simulate decrement X from Y using increment operation
 936                  * we need to add UINT32_MAX X times to Y.
 937                  * Each adding of UINT32_MAX decrements Y by 1.
 938                  */
 939                 value *= UINT32_MAX;
 940         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 941         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 942         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 943         item.spec = &tcp;
 944         item.mask = &tcp_mask;
 945         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 946                                              MLX5_MODIFICATION_TYPE_ADD, error);
 947 }
 948
 949 static enum mlx5_modification_field reg_to_field[] = {
 950         [REG_NON] = MLX5_MODI_OUT_NONE,
 951         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 952         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 953         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 954         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 955         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 956         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 957         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 958         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 959         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 960         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 961 };
 962
 963 /**
 964  * Convert register set to DV specification.
 965  *
 966  * @param[in,out] resource
 967  *   Pointer to the modify-header resource.
 968  * @param[in] action
 969  *   Pointer to action specification.
 970  * @param[out] error
 971  *   Pointer to the error structure.
 972  *
 973  * @return
 974  *   0 on success, a negative errno value otherwise and rte_errno is set.
 975  */
 976 static int
 977 flow_dv_convert_action_set_reg
 978                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 979                          const struct rte_flow_action *action,
 980                          struct rte_flow_error *error)
 981 {
 982         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 983         struct mlx5_modification_cmd *actions = resource->actions;
 984         uint32_t i = resource->actions_num;
 985
 986         if (i >= MLX5_MAX_MODIFY_NUM)
 987                 return rte_flow_error_set(error, EINVAL,
 988                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 989                                           "too many items to modify");
 990         MLX5_ASSERT(conf->id != REG_NON);
 991         MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
 992         actions[i] = (struct mlx5_modification_cmd) {
 993                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 994                 .field = reg_to_field[conf->id],
 995                 .offset = conf->offset,
 996                 .length = conf->length,
 997         };
 998         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 999         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1000         ++i;
1001         resource->actions_num = i;
1002         return 0;
1003 }
1004
1005 /**
1006  * Convert SET_TAG action to DV specification.
1007  *
1008  * @param[in] dev
1009  *   Pointer to the rte_eth_dev structure.
1010  * @param[in,out] resource
1011  *   Pointer to the modify-header resource.
1012  * @param[in] conf
1013  *   Pointer to action specification.
1014  * @param[out] error
1015  *   Pointer to the error structure.
1016  *
1017  * @return
1018  *   0 on success, a negative errno value otherwise and rte_errno is set.
1019  */
1020 static int
1021 flow_dv_convert_action_set_tag
1022                         (struct rte_eth_dev *dev,
1023                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1024                          const struct rte_flow_action_set_tag *conf,
1025                          struct rte_flow_error *error)
1026 {
1027         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1028         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1029         struct rte_flow_item item = {
1030                 .spec = &data,
1031                 .mask = &mask,
1032         };
1033         struct field_modify_info reg_c_x[] = {
1034                 [1] = {0, 0, 0},
1035         };
1036         enum mlx5_modification_field reg_type;
1037         int ret;
1038
1039         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1040         if (ret < 0)
1041                 return ret;
1042         MLX5_ASSERT(ret != REG_NON);
1043         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1044         reg_type = reg_to_field[ret];
1045         MLX5_ASSERT(reg_type > 0);
1046         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1047         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1048                                              MLX5_MODIFICATION_TYPE_SET, error);
1049 }
1050
1051 /**
1052  * Convert internal COPY_REG action to DV specification.
1053  *
1054  * @param[in] dev
1055  *   Pointer to the rte_eth_dev structure.
1056  * @param[in,out] res
1057  *   Pointer to the modify-header resource.
1058  * @param[in] action
1059  *   Pointer to action specification.
1060  * @param[out] error
1061  *   Pointer to the error structure.
1062  *
1063  * @return
1064  *   0 on success, a negative errno value otherwise and rte_errno is set.
1065  */
1066 static int
1067 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1068                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1069                                  const struct rte_flow_action *action,
1070                                  struct rte_flow_error *error)
1071 {
1072         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1073         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1074         struct rte_flow_item item = {
1075                 .spec = NULL,
1076                 .mask = &mask,
1077         };
1078         struct field_modify_info reg_src[] = {
1079                 {4, 0, reg_to_field[conf->src]},
1080                 {0, 0, 0},
1081         };
1082         struct field_modify_info reg_dst = {
1083                 .offset = 0,
1084                 .id = reg_to_field[conf->dst],
1085         };
1086         /* Adjust reg_c[0] usage according to reported mask. */
1087         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1088                 struct mlx5_priv *priv = dev->data->dev_private;
1089                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1090
1091                 MLX5_ASSERT(reg_c0);
1092                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1093                 if (conf->dst == REG_C_0) {
1094                         /* Copy to reg_c[0], within mask only. */
1095                         reg_dst.offset = rte_bsf32(reg_c0);
1096                         /*
1097                          * Mask is ignoring the enianness, because
1098                          * there is no conversion in datapath.
1099                          */
1100 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1101                         /* Copy from destination lower bits to reg_c[0]. */
1102                         mask = reg_c0 >> reg_dst.offset;
1103 #else
1104                         /* Copy from destination upper bits to reg_c[0]. */
1105                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1106                                           rte_fls_u32(reg_c0));
1107 #endif
1108                 } else {
1109                         mask = rte_cpu_to_be_32(reg_c0);
1110 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1111                         /* Copy from reg_c[0] to destination lower bits. */
1112                         reg_dst.offset = 0;
1113 #else
1114                         /* Copy from reg_c[0] to destination upper bits. */
1115                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1116                                          (rte_fls_u32(reg_c0) -
1117                                           rte_bsf32(reg_c0));
1118 #endif
1119                 }
1120         }
1121         return flow_dv_convert_modify_action(&item,
1122                                              reg_src, &reg_dst, res,
1123                                              MLX5_MODIFICATION_TYPE_COPY,
1124                                              error);
1125 }
1126
1127 /**
1128  * Convert MARK action to DV specification. This routine is used
1129  * in extensive metadata only and requires metadata register to be
1130  * handled. In legacy mode hardware tag resource is engaged.
1131  *
1132  * @param[in] dev
1133  *   Pointer to the rte_eth_dev structure.
1134  * @param[in] conf
1135  *   Pointer to MARK action specification.
1136  * @param[in,out] resource
1137  *   Pointer to the modify-header resource.
1138  * @param[out] error
1139  *   Pointer to the error structure.
1140  *
1141  * @return
1142  *   0 on success, a negative errno value otherwise and rte_errno is set.
1143  */
1144 static int
1145 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1146                             const struct rte_flow_action_mark *conf,
1147                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1148                             struct rte_flow_error *error)
1149 {
1150         struct mlx5_priv *priv = dev->data->dev_private;
1151         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1152                                            priv->sh->dv_mark_mask);
1153         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1154         struct rte_flow_item item = {
1155                 .spec = &data,
1156                 .mask = &mask,
1157         };
1158         struct field_modify_info reg_c_x[] = {
1159                 [1] = {0, 0, 0},
1160         };
1161         int reg;
1162
1163         if (!mask)
1164                 return rte_flow_error_set(error, EINVAL,
1165                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1166                                           NULL, "zero mark action mask");
1167         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1168         if (reg < 0)
1169                 return reg;
1170         MLX5_ASSERT(reg > 0);
1171         if (reg == REG_C_0) {
1172                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1173                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1174
1175                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1176                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1177                 mask = rte_cpu_to_be_32(mask << shl_c0);
1178         }
1179         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1180         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1181                                              MLX5_MODIFICATION_TYPE_SET, error);
1182 }
1183
1184 /**
1185  * Get metadata register index for specified steering domain.
1186  *
1187  * @param[in] dev
1188  *   Pointer to the rte_eth_dev structure.
1189  * @param[in] attr
1190  *   Attributes of flow to determine steering domain.
1191  * @param[out] error
1192  *   Pointer to the error structure.
1193  *
1194  * @return
1195  *   positive index on success, a negative errno value otherwise
1196  *   and rte_errno is set.
1197  */
1198 static enum modify_reg
1199 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1200                          const struct rte_flow_attr *attr,
1201                          struct rte_flow_error *error)
1202 {
1203         int reg =
1204                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1205                                           MLX5_METADATA_FDB :
1206                                             attr->egress ?
1207                                             MLX5_METADATA_TX :
1208                                             MLX5_METADATA_RX, 0, error);
1209         if (reg < 0)
1210                 return rte_flow_error_set(error,
1211                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1212                                           NULL, "unavailable "
1213                                           "metadata register");
1214         return reg;
1215 }
1216
1217 /**
1218  * Convert SET_META action to DV specification.
1219  *
1220  * @param[in] dev
1221  *   Pointer to the rte_eth_dev structure.
1222  * @param[in,out] resource
1223  *   Pointer to the modify-header resource.
1224  * @param[in] attr
1225  *   Attributes of flow that includes this item.
1226  * @param[in] conf
1227  *   Pointer to action specification.
1228  * @param[out] error
1229  *   Pointer to the error structure.
1230  *
1231  * @return
1232  *   0 on success, a negative errno value otherwise and rte_errno is set.
1233  */
1234 static int
1235 flow_dv_convert_action_set_meta
1236                         (struct rte_eth_dev *dev,
1237                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1238                          const struct rte_flow_attr *attr,
1239                          const struct rte_flow_action_set_meta *conf,
1240                          struct rte_flow_error *error)
1241 {
1242         uint32_t data = conf->data;
1243         uint32_t mask = conf->mask;
1244         struct rte_flow_item item = {
1245                 .spec = &data,
1246                 .mask = &mask,
1247         };
1248         struct field_modify_info reg_c_x[] = {
1249                 [1] = {0, 0, 0},
1250         };
1251         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1252
1253         if (reg < 0)
1254                 return reg;
1255         MLX5_ASSERT(reg != REG_NON);
1256         /*
1257          * In datapath code there is no endianness
1258          * coversions for perfromance reasons, all
1259          * pattern conversions are done in rte_flow.
1260          */
1261         if (reg == REG_C_0) {
1262                 struct mlx5_priv *priv = dev->data->dev_private;
1263                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1264                 uint32_t shl_c0;
1265
1266                 MLX5_ASSERT(msk_c0);
1267 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1268                 shl_c0 = rte_bsf32(msk_c0);
1269 #else
1270                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1271 #endif
1272                 mask <<= shl_c0;
1273                 data <<= shl_c0;
1274                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1275         }
1276         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1277         /* The routine expects parameters in memory as big-endian ones. */
1278         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1279                                              MLX5_MODIFICATION_TYPE_SET, error);
1280 }
1281
1282 /**
1283  * Convert modify-header set IPv4 DSCP action to DV specification.
1284  *
1285  * @param[in,out] resource
1286  *   Pointer to the modify-header resource.
1287  * @param[in] action
1288  *   Pointer to action specification.
1289  * @param[out] error
1290  *   Pointer to the error structure.
1291  *
1292  * @return
1293  *   0 on success, a negative errno value otherwise and rte_errno is set.
1294  */
1295 static int
1296 flow_dv_convert_action_modify_ipv4_dscp
1297                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1298                          const struct rte_flow_action *action,
1299                          struct rte_flow_error *error)
1300 {
1301         const struct rte_flow_action_set_dscp *conf =
1302                 (const struct rte_flow_action_set_dscp *)(action->conf);
1303         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1304         struct rte_flow_item_ipv4 ipv4;
1305         struct rte_flow_item_ipv4 ipv4_mask;
1306
1307         memset(&ipv4, 0, sizeof(ipv4));
1308         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1309         ipv4.hdr.type_of_service = conf->dscp;
1310         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1311         item.spec = &ipv4;
1312         item.mask = &ipv4_mask;
1313         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1314                                              MLX5_MODIFICATION_TYPE_SET, error);
1315 }
1316
1317 /**
1318  * Convert modify-header set IPv6 DSCP action to DV specification.
1319  *
1320  * @param[in,out] resource
1321  *   Pointer to the modify-header resource.
1322  * @param[in] action
1323  *   Pointer to action specification.
1324  * @param[out] error
1325  *   Pointer to the error structure.
1326  *
1327  * @return
1328  *   0 on success, a negative errno value otherwise and rte_errno is set.
1329  */
1330 static int
1331 flow_dv_convert_action_modify_ipv6_dscp
1332                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1333                          const struct rte_flow_action *action,
1334                          struct rte_flow_error *error)
1335 {
1336         const struct rte_flow_action_set_dscp *conf =
1337                 (const struct rte_flow_action_set_dscp *)(action->conf);
1338         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1339         struct rte_flow_item_ipv6 ipv6;
1340         struct rte_flow_item_ipv6 ipv6_mask;
1341
1342         memset(&ipv6, 0, sizeof(ipv6));
1343         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1344         /*
1345          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1346          * rdma-core only accept the DSCP bits byte aligned start from
1347          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1348          * bits in IPv6 case as rdma-core requires byte aligned value.
1349          */
1350         ipv6.hdr.vtc_flow = conf->dscp;
1351         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1352         item.spec = &ipv6;
1353         item.mask = &ipv6_mask;
1354         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1355                                              MLX5_MODIFICATION_TYPE_SET, error);
1356 }
1357
1358 static int
1359 mlx5_flow_item_field_width(struct mlx5_dev_config *config,
1360                            enum rte_flow_field_id field)
1361 {
1362         switch (field) {
1363         case RTE_FLOW_FIELD_START:
1364                 return 32;
1365         case RTE_FLOW_FIELD_MAC_DST:
1366         case RTE_FLOW_FIELD_MAC_SRC:
1367                 return 48;
1368         case RTE_FLOW_FIELD_VLAN_TYPE:
1369                 return 16;
1370         case RTE_FLOW_FIELD_VLAN_ID:
1371                 return 12;
1372         case RTE_FLOW_FIELD_MAC_TYPE:
1373                 return 16;
1374         case RTE_FLOW_FIELD_IPV4_DSCP:
1375                 return 6;
1376         case RTE_FLOW_FIELD_IPV4_TTL:
1377                 return 8;
1378         case RTE_FLOW_FIELD_IPV4_SRC:
1379         case RTE_FLOW_FIELD_IPV4_DST:
1380                 return 32;
1381         case RTE_FLOW_FIELD_IPV6_DSCP:
1382                 return 6;
1383         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1384                 return 8;
1385         case RTE_FLOW_FIELD_IPV6_SRC:
1386         case RTE_FLOW_FIELD_IPV6_DST:
1387                 return 128;
1388         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1389         case RTE_FLOW_FIELD_TCP_PORT_DST:
1390                 return 16;
1391         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1392         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1393                 return 32;
1394         case RTE_FLOW_FIELD_TCP_FLAGS:
1395                 return 9;
1396         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1397         case RTE_FLOW_FIELD_UDP_PORT_DST:
1398                 return 16;
1399         case RTE_FLOW_FIELD_VXLAN_VNI:
1400         case RTE_FLOW_FIELD_GENEVE_VNI:
1401                 return 24;
1402         case RTE_FLOW_FIELD_GTP_TEID:
1403         case RTE_FLOW_FIELD_TAG:
1404                 return 32;
1405         case RTE_FLOW_FIELD_MARK:
1406                 return 24;
1407         case RTE_FLOW_FIELD_META:
1408                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_META16)
1409                         return 16;
1410                 else if (config->dv_xmeta_en == MLX5_XMETA_MODE_META32)
1411                         return 32;
1412                 else
1413                         return 0;
1414         case RTE_FLOW_FIELD_POINTER:
1415         case RTE_FLOW_FIELD_VALUE:
1416                 return 64;
1417         default:
1418                 MLX5_ASSERT(false);
1419         }
1420         return 0;
1421 }
1422
1423 static void
1424 mlx5_flow_field_id_to_modify_info
1425                 (const struct rte_flow_action_modify_data *data,
1426                  struct field_modify_info *info,
1427                  uint32_t *mask, uint32_t *value,
1428                  uint32_t width, uint32_t dst_width,
1429                  struct rte_eth_dev *dev,
1430                  const struct rte_flow_attr *attr,
1431                  struct rte_flow_error *error)
1432 {
1433         struct mlx5_priv *priv = dev->data->dev_private;
1434         struct mlx5_dev_config *config = &priv->config;
1435         uint32_t idx = 0;
1436         uint64_t val = 0;
1437         switch (data->field) {
1438         case RTE_FLOW_FIELD_START:
1439                 /* not supported yet */
1440                 MLX5_ASSERT(false);
1441                 break;
1442         case RTE_FLOW_FIELD_MAC_DST:
1443                 if (mask) {
1444                         if (data->offset < 32) {
1445                                 info[idx] = (struct field_modify_info){4, 0,
1446                                                 MLX5_MODI_OUT_DMAC_47_16};
1447                                 if (width < 32) {
1448                                         mask[idx] =
1449                                                 rte_cpu_to_be_32(0xffffffff >>
1450                                                                  (32 - width));
1451                                         width = 0;
1452                                 } else {
1453                                         mask[idx] = RTE_BE32(0xffffffff);
1454                                         width -= 32;
1455                                 }
1456                                 if (!width)
1457                                         break;
1458                                 ++idx;
1459                         }
1460                         info[idx] = (struct field_modify_info){2, 4 * idx,
1461                                                 MLX5_MODI_OUT_DMAC_15_0};
1462                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1463                 } else {
1464                         if (data->offset < 32)
1465                                 info[idx++] = (struct field_modify_info){4, 0,
1466                                                 MLX5_MODI_OUT_DMAC_47_16};
1467                         info[idx] = (struct field_modify_info){2, 0,
1468                                                 MLX5_MODI_OUT_DMAC_15_0};
1469                 }
1470                 break;
1471         case RTE_FLOW_FIELD_MAC_SRC:
1472                 if (mask) {
1473                         if (data->offset < 32) {
1474                                 info[idx] = (struct field_modify_info){4, 0,
1475                                                 MLX5_MODI_OUT_SMAC_47_16};
1476                                 if (width < 32) {
1477                                         mask[idx] =
1478                                                 rte_cpu_to_be_32(0xffffffff >>
1479                                                                 (32 - width));
1480                                         width = 0;
1481                                 } else {
1482                                         mask[idx] = RTE_BE32(0xffffffff);
1483                                         width -= 32;
1484                                 }
1485                                 if (!width)
1486                                         break;
1487                                 ++idx;
1488                         }
1489                         info[idx] = (struct field_modify_info){2, 4 * idx,
1490                                                 MLX5_MODI_OUT_SMAC_15_0};
1491                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1492                 } else {
1493                         if (data->offset < 32)
1494                                 info[idx++] = (struct field_modify_info){4, 0,
1495                                                 MLX5_MODI_OUT_SMAC_47_16};
1496                         info[idx] = (struct field_modify_info){2, 0,
1497                                                 MLX5_MODI_OUT_SMAC_15_0};
1498                 }
1499                 break;
1500         case RTE_FLOW_FIELD_VLAN_TYPE:
1501                 /* not supported yet */
1502                 break;
1503         case RTE_FLOW_FIELD_VLAN_ID:
1504                 info[idx] = (struct field_modify_info){2, 0,
1505                                         MLX5_MODI_OUT_FIRST_VID};
1506                 if (mask)
1507                         mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
1508                 break;
1509         case RTE_FLOW_FIELD_MAC_TYPE:
1510                 info[idx] = (struct field_modify_info){2, 0,
1511                                         MLX5_MODI_OUT_ETHERTYPE};
1512                 if (mask)
1513                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1514                 break;
1515         case RTE_FLOW_FIELD_IPV4_DSCP:
1516                 info[idx] = (struct field_modify_info){1, 0,
1517                                         MLX5_MODI_OUT_IP_DSCP};
1518                 if (mask)
1519                         mask[idx] = 0x3f >> (6 - width);
1520                 break;
1521         case RTE_FLOW_FIELD_IPV4_TTL:
1522                 info[idx] = (struct field_modify_info){1, 0,
1523                                         MLX5_MODI_OUT_IPV4_TTL};
1524                 if (mask)
1525                         mask[idx] = 0xff >> (8 - width);
1526                 break;
1527         case RTE_FLOW_FIELD_IPV4_SRC:
1528                 info[idx] = (struct field_modify_info){4, 0,
1529                                         MLX5_MODI_OUT_SIPV4};
1530                 if (mask)
1531                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1532                                                      (32 - width));
1533                 break;
1534         case RTE_FLOW_FIELD_IPV4_DST:
1535                 info[idx] = (struct field_modify_info){4, 0,
1536                                         MLX5_MODI_OUT_DIPV4};
1537                 if (mask)
1538                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1539                                                      (32 - width));
1540                 break;
1541         case RTE_FLOW_FIELD_IPV6_DSCP:
1542                 info[idx] = (struct field_modify_info){1, 0,
1543                                         MLX5_MODI_OUT_IP_DSCP};
1544                 if (mask)
1545                         mask[idx] = 0x3f >> (6 - width);
1546                 break;
1547         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1548                 info[idx] = (struct field_modify_info){1, 0,
1549                                         MLX5_MODI_OUT_IPV6_HOPLIMIT};
1550                 if (mask)
1551                         mask[idx] = 0xff >> (8 - width);
1552                 break;
1553         case RTE_FLOW_FIELD_IPV6_SRC:
1554                 if (mask) {
1555                         if (data->offset < 32) {
1556                                 info[idx] = (struct field_modify_info){4,
1557                                                 4 * idx,
1558                                                 MLX5_MODI_OUT_SIPV6_31_0};
1559                                 if (width < 32) {
1560                                         mask[idx] =
1561                                                 rte_cpu_to_be_32(0xffffffff >>
1562                                                                  (32 - width));
1563                                         width = 0;
1564                                 } else {
1565                                         mask[idx] = RTE_BE32(0xffffffff);
1566                                         width -= 32;
1567                                 }
1568                                 if (!width)
1569                                         break;
1570                                 ++idx;
1571                         }
1572                         if (data->offset < 64) {
1573                                 info[idx] = (struct field_modify_info){4,
1574                                                 4 * idx,
1575                                                 MLX5_MODI_OUT_SIPV6_63_32};
1576                                 if (width < 32) {
1577                                         mask[idx] =
1578                                                 rte_cpu_to_be_32(0xffffffff >>
1579                                                                  (32 - width));
1580                                         width = 0;
1581                                 } else {
1582                                         mask[idx] = RTE_BE32(0xffffffff);
1583                                         width -= 32;
1584                                 }
1585                                 if (!width)
1586                                         break;
1587                                 ++idx;
1588                         }
1589                         if (data->offset < 96) {
1590                                 info[idx] = (struct field_modify_info){4,
1591                                                 4 * idx,
1592                                                 MLX5_MODI_OUT_SIPV6_95_64};
1593                                 if (width < 32) {
1594                                         mask[idx] =
1595                                                 rte_cpu_to_be_32(0xffffffff >>
1596                                                                  (32 - width));
1597                                         width = 0;
1598                                 } else {
1599                                         mask[idx] = RTE_BE32(0xffffffff);
1600                                         width -= 32;
1601                                 }
1602                                 if (!width)
1603                                         break;
1604                                 ++idx;
1605                         }
1606                         info[idx] = (struct field_modify_info){4, 4 * idx,
1607                                                 MLX5_MODI_OUT_SIPV6_127_96};
1608                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1609                                                      (32 - width));
1610                 } else {
1611                         if (data->offset < 32)
1612                                 info[idx++] = (struct field_modify_info){4, 0,
1613                                                 MLX5_MODI_OUT_SIPV6_31_0};
1614                         if (data->offset < 64)
1615                                 info[idx++] = (struct field_modify_info){4, 0,
1616                                                 MLX5_MODI_OUT_SIPV6_63_32};
1617                         if (data->offset < 96)
1618                                 info[idx++] = (struct field_modify_info){4, 0,
1619                                                 MLX5_MODI_OUT_SIPV6_95_64};
1620                         if (data->offset < 128)
1621                                 info[idx++] = (struct field_modify_info){4, 0,
1622                                                 MLX5_MODI_OUT_SIPV6_127_96};
1623                 }
1624                 break;
1625         case RTE_FLOW_FIELD_IPV6_DST:
1626                 if (mask) {
1627                         if (data->offset < 32) {
1628                                 info[idx] = (struct field_modify_info){4,
1629                                                 4 * idx,
1630                                                 MLX5_MODI_OUT_DIPV6_31_0};
1631                                 if (width < 32) {
1632                                         mask[idx] =
1633                                                 rte_cpu_to_be_32(0xffffffff >>
1634                                                                  (32 - width));
1635                                         width = 0;
1636                                 } else {
1637                                         mask[idx] = RTE_BE32(0xffffffff);
1638                                         width -= 32;
1639                                 }
1640                                 if (!width)
1641                                         break;
1642                                 ++idx;
1643                         }
1644                         if (data->offset < 64) {
1645                                 info[idx] = (struct field_modify_info){4,
1646                                                 4 * idx,
1647                                                 MLX5_MODI_OUT_DIPV6_63_32};
1648                                 if (width < 32) {
1649                                         mask[idx] =
1650                                                 rte_cpu_to_be_32(0xffffffff >>
1651                                                                  (32 - width));
1652                                         width = 0;
1653                                 } else {
1654                                         mask[idx] = RTE_BE32(0xffffffff);
1655                                         width -= 32;
1656                                 }
1657                                 if (!width)
1658                                         break;
1659                                 ++idx;
1660                         }
1661                         if (data->offset < 96) {
1662                                 info[idx] = (struct field_modify_info){4,
1663                                                 4 * idx,
1664                                                 MLX5_MODI_OUT_DIPV6_95_64};
1665                                 if (width < 32) {
1666                                         mask[idx] =
1667                                                 rte_cpu_to_be_32(0xffffffff >>
1668                                                                  (32 - width));
1669                                         width = 0;
1670                                 } else {
1671                                         mask[idx] = RTE_BE32(0xffffffff);
1672                                         width -= 32;
1673                                 }
1674                                 if (!width)
1675                                         break;
1676                                 ++idx;
1677                         }
1678                         info[idx] = (struct field_modify_info){4, 4 * idx,
1679                                                 MLX5_MODI_OUT_DIPV6_127_96};
1680                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1681                                                      (32 - width));
1682                 } else {
1683                         if (data->offset < 32)
1684                                 info[idx++] = (struct field_modify_info){4, 0,
1685                                                 MLX5_MODI_OUT_DIPV6_31_0};
1686                         if (data->offset < 64)
1687                                 info[idx++] = (struct field_modify_info){4, 0,
1688                                                 MLX5_MODI_OUT_DIPV6_63_32};
1689                         if (data->offset < 96)
1690                                 info[idx++] = (struct field_modify_info){4, 0,
1691                                                 MLX5_MODI_OUT_DIPV6_95_64};
1692                         if (data->offset < 128)
1693                                 info[idx++] = (struct field_modify_info){4, 0,
1694                                                 MLX5_MODI_OUT_DIPV6_127_96};
1695                 }
1696                 break;
1697         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1698                 info[idx] = (struct field_modify_info){2, 0,
1699                                         MLX5_MODI_OUT_TCP_SPORT};
1700                 if (mask)
1701                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1702                 break;
1703         case RTE_FLOW_FIELD_TCP_PORT_DST:
1704                 info[idx] = (struct field_modify_info){2, 0,
1705                                         MLX5_MODI_OUT_TCP_DPORT};
1706                 if (mask)
1707                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1708                 break;
1709         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1710                 info[idx] = (struct field_modify_info){4, 0,
1711                                         MLX5_MODI_OUT_TCP_SEQ_NUM};
1712                 if (mask)
1713                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1714                                                      (32 - width));
1715                 break;
1716         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1717                 info[idx] = (struct field_modify_info){4, 0,
1718                                         MLX5_MODI_OUT_TCP_ACK_NUM};
1719                 if (mask)
1720                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1721                                                      (32 - width));
1722                 break;
1723         case RTE_FLOW_FIELD_TCP_FLAGS:
1724                 info[idx] = (struct field_modify_info){2, 0,
1725                                         MLX5_MODI_OUT_TCP_FLAGS};
1726                 if (mask)
1727                         mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
1728                 break;
1729         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1730                 info[idx] = (struct field_modify_info){2, 0,
1731                                         MLX5_MODI_OUT_UDP_SPORT};
1732                 if (mask)
1733                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1734                 break;
1735         case RTE_FLOW_FIELD_UDP_PORT_DST:
1736                 info[idx] = (struct field_modify_info){2, 0,
1737                                         MLX5_MODI_OUT_UDP_DPORT};
1738                 if (mask)
1739                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1740                 break;
1741         case RTE_FLOW_FIELD_VXLAN_VNI:
1742                 /* not supported yet */
1743                 break;
1744         case RTE_FLOW_FIELD_GENEVE_VNI:
1745                 /* not supported yet*/
1746                 break;
1747         case RTE_FLOW_FIELD_GTP_TEID:
1748                 info[idx] = (struct field_modify_info){4, 0,
1749                                         MLX5_MODI_GTP_TEID};
1750                 if (mask)
1751                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1752                                                      (32 - width));
1753                 break;
1754         case RTE_FLOW_FIELD_TAG:
1755                 {
1756                         int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
1757                                                    data->level, error);
1758                         if (reg < 0)
1759                                 return;
1760                         MLX5_ASSERT(reg != REG_NON);
1761                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1762                         info[idx] = (struct field_modify_info){4, 0,
1763                                                 reg_to_field[reg]};
1764                         if (mask)
1765                                 mask[idx] =
1766                                         rte_cpu_to_be_32(0xffffffff >>
1767                                                          (32 - width));
1768                 }
1769                 break;
1770         case RTE_FLOW_FIELD_MARK:
1771                 {
1772                         int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
1773                                                        0, error);
1774                         if (reg < 0)
1775                                 return;
1776                         MLX5_ASSERT(reg != REG_NON);
1777                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1778                         info[idx] = (struct field_modify_info){4, 0,
1779                                                 reg_to_field[reg]};
1780                         if (mask)
1781                                 mask[idx] =
1782                                         rte_cpu_to_be_32(0xffffffff >>
1783                                                          (32 - width));
1784                 }
1785                 break;
1786         case RTE_FLOW_FIELD_META:
1787                 {
1788                         unsigned int xmeta = config->dv_xmeta_en;
1789                         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1790                         if (reg < 0)
1791                                 return;
1792                         MLX5_ASSERT(reg != REG_NON);
1793                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1794                         if (xmeta == MLX5_XMETA_MODE_META16) {
1795                                 info[idx] = (struct field_modify_info){2, 0,
1796                                                         reg_to_field[reg]};
1797                                 if (mask)
1798                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1799                                                                 (16 - width));
1800                         } else if (xmeta == MLX5_XMETA_MODE_META32) {
1801                                 info[idx] = (struct field_modify_info){4, 0,
1802                                                         reg_to_field[reg]};
1803                                 if (mask)
1804                                         mask[idx] =
1805                                                 rte_cpu_to_be_32(0xffffffff >>
1806                                                                 (32 - width));
1807                         } else {
1808                                 MLX5_ASSERT(false);
1809                         }
1810                 }
1811                 break;
1812         case RTE_FLOW_FIELD_POINTER:
1813         case RTE_FLOW_FIELD_VALUE:
1814                 if (data->field == RTE_FLOW_FIELD_POINTER)
1815                         memcpy(&val, (void *)(uintptr_t)data->value,
1816                                sizeof(uint64_t));
1817                 else
1818                         val = data->value;
1819                 for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) {
1820                         if (mask[idx]) {
1821                                 if (dst_width > 16) {
1822                                         value[idx] = rte_cpu_to_be_32(val);
1823                                         val >>= 32;
1824                                 } else if (dst_width > 8) {
1825                                         value[idx] = rte_cpu_to_be_16(val);
1826                                         val >>= 16;
1827                                 } else {
1828                                         value[idx] = (uint8_t)val;
1829                                         val >>= 8;
1830                                 }
1831                                 if (!val)
1832                                         break;
1833                         }
1834                 }
1835                 break;
1836         default:
1837                 MLX5_ASSERT(false);
1838                 break;
1839         }
1840 }
1841
1842 /**
1843  * Convert modify_field action to DV specification.
1844  *
1845  * @param[in] dev
1846  *   Pointer to the rte_eth_dev structure.
1847  * @param[in,out] resource
1848  *   Pointer to the modify-header resource.
1849  * @param[in] action
1850  *   Pointer to action specification.
1851  * @param[in] attr
1852  *   Attributes of flow that includes this item.
1853  * @param[out] error
1854  *   Pointer to the error structure.
1855  *
1856  * @return
1857  *   0 on success, a negative errno value otherwise and rte_errno is set.
1858  */
1859 static int
1860 flow_dv_convert_action_modify_field
1861                         (struct rte_eth_dev *dev,
1862                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1863                          const struct rte_flow_action *action,
1864                          const struct rte_flow_attr *attr,
1865                          struct rte_flow_error *error)
1866 {
1867         struct mlx5_priv *priv = dev->data->dev_private;
1868         struct mlx5_dev_config *config = &priv->config;
1869         const struct rte_flow_action_modify_field *conf =
1870                 (const struct rte_flow_action_modify_field *)(action->conf);
1871         struct rte_flow_item item;
1872         struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
1873                                                                 {0, 0, 0} };
1874         struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
1875                                                                 {0, 0, 0} };
1876         uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1877         uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1878         uint32_t type;
1879         uint32_t dst_width = mlx5_flow_item_field_width(config,
1880                                                         conf->dst.field);
1881
1882         if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
1883                 conf->src.field == RTE_FLOW_FIELD_VALUE) {
1884                 type = MLX5_MODIFICATION_TYPE_SET;
1885                 /** For SET fill the destination field (field) first. */
1886                 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
1887                         value, conf->width, dst_width, dev, attr, error);
1888                 /** Then copy immediate value from source as per mask. */
1889                 mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask,
1890                         value, conf->width, dst_width, dev, attr, error);
1891                 item.spec = &value;
1892         } else {
1893                 type = MLX5_MODIFICATION_TYPE_COPY;
1894                 /** For COPY fill the destination field (dcopy) without mask. */
1895                 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
1896                         value, conf->width, dst_width, dev, attr, error);
1897                 /** Then construct the source field (field) with mask. */
1898                 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
1899                         value, conf->width, dst_width, dev, attr, error);
1900         }
1901         item.mask = &mask;
1902         return flow_dv_convert_modify_action(&item,
1903                         field, dcopy, resource, type, error);
1904 }
1905
1906 /**
1907  * Validate MARK item.
1908  *
1909  * @param[in] dev
1910  *   Pointer to the rte_eth_dev structure.
1911  * @param[in] item
1912  *   Item specification.
1913  * @param[in] attr
1914  *   Attributes of flow that includes this item.
1915  * @param[out] error
1916  *   Pointer to error structure.
1917  *
1918  * @return
1919  *   0 on success, a negative errno value otherwise and rte_errno is set.
1920  */
1921 static int
1922 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1923                            const struct rte_flow_item *item,
1924                            const struct rte_flow_attr *attr __rte_unused,
1925                            struct rte_flow_error *error)
1926 {
1927         struct mlx5_priv *priv = dev->data->dev_private;
1928         struct mlx5_dev_config *config = &priv->config;
1929         const struct rte_flow_item_mark *spec = item->spec;
1930         const struct rte_flow_item_mark *mask = item->mask;
1931         const struct rte_flow_item_mark nic_mask = {
1932                 .id = priv->sh->dv_mark_mask,
1933         };
1934         int ret;
1935
1936         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1937                 return rte_flow_error_set(error, ENOTSUP,
1938                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1939                                           "extended metadata feature"
1940                                           " isn't enabled");
1941         if (!mlx5_flow_ext_mreg_supported(dev))
1942                 return rte_flow_error_set(error, ENOTSUP,
1943                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1944                                           "extended metadata register"
1945                                           " isn't supported");
1946         if (!nic_mask.id)
1947                 return rte_flow_error_set(error, ENOTSUP,
1948                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1949                                           "extended metadata register"
1950                                           " isn't available");
1951         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1952         if (ret < 0)
1953                 return ret;
1954         if (!spec)
1955                 return rte_flow_error_set(error, EINVAL,
1956                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1957                                           item->spec,
1958                                           "data cannot be empty");
1959         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1960                 return rte_flow_error_set(error, EINVAL,
1961                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1962                                           &spec->id,
1963                                           "mark id exceeds the limit");
1964         if (!mask)
1965                 mask = &nic_mask;
1966         if (!mask->id)
1967                 return rte_flow_error_set(error, EINVAL,
1968                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1969                                         "mask cannot be zero");
1970
1971         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1972                                         (const uint8_t *)&nic_mask,
1973                                         sizeof(struct rte_flow_item_mark),
1974                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1975         if (ret < 0)
1976                 return ret;
1977         return 0;
1978 }
1979
1980 /**
1981  * Validate META item.
1982  *
1983  * @param[in] dev
1984  *   Pointer to the rte_eth_dev structure.
1985  * @param[in] item
1986  *   Item specification.
1987  * @param[in] attr
1988  *   Attributes of flow that includes this item.
1989  * @param[out] error
1990  *   Pointer to error structure.
1991  *
1992  * @return
1993  *   0 on success, a negative errno value otherwise and rte_errno is set.
1994  */
1995 static int
1996 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1997                            const struct rte_flow_item *item,
1998                            const struct rte_flow_attr *attr,
1999                            struct rte_flow_error *error)
2000 {
2001         struct mlx5_priv *priv = dev->data->dev_private;
2002         struct mlx5_dev_config *config = &priv->config;
2003         const struct rte_flow_item_meta *spec = item->spec;
2004         const struct rte_flow_item_meta *mask = item->mask;
2005         struct rte_flow_item_meta nic_mask = {
2006                 .data = UINT32_MAX
2007         };
2008         int reg;
2009         int ret;
2010
2011         if (!spec)
2012                 return rte_flow_error_set(error, EINVAL,
2013                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2014                                           item->spec,
2015                                           "data cannot be empty");
2016         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2017                 if (!mlx5_flow_ext_mreg_supported(dev))
2018                         return rte_flow_error_set(error, ENOTSUP,
2019                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2020                                           "extended metadata register"
2021                                           " isn't supported");
2022                 reg = flow_dv_get_metadata_reg(dev, attr, error);
2023                 if (reg < 0)
2024                         return reg;
2025                 if (reg == REG_NON)
2026                         return rte_flow_error_set(error, ENOTSUP,
2027                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2028                                         "unavalable extended metadata register");
2029                 if (reg == REG_B)
2030                         return rte_flow_error_set(error, ENOTSUP,
2031                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2032                                           "match on reg_b "
2033                                           "isn't supported");
2034                 if (reg != REG_A)
2035                         nic_mask.data = priv->sh->dv_meta_mask;
2036         } else {
2037                 if (attr->transfer)
2038                         return rte_flow_error_set(error, ENOTSUP,
2039                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2040                                         "extended metadata feature "
2041                                         "should be enabled when "
2042                                         "meta item is requested "
2043                                         "with e-switch mode ");
2044                 if (attr->ingress)
2045                         return rte_flow_error_set(error, ENOTSUP,
2046                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2047                                         "match on metadata for ingress "
2048                                         "is not supported in legacy "
2049                                         "metadata mode");
2050         }
2051         if (!mask)
2052                 mask = &rte_flow_item_meta_mask;
2053         if (!mask->data)
2054                 return rte_flow_error_set(error, EINVAL,
2055                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2056                                         "mask cannot be zero");
2057
2058         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2059                                         (const uint8_t *)&nic_mask,
2060                                         sizeof(struct rte_flow_item_meta),
2061                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2062         return ret;
2063 }
2064
2065 /**
2066  * Validate TAG item.
2067  *
2068  * @param[in] dev
2069  *   Pointer to the rte_eth_dev structure.
2070  * @param[in] item
2071  *   Item specification.
2072  * @param[in] attr
2073  *   Attributes of flow that includes this item.
2074  * @param[out] error
2075  *   Pointer to error structure.
2076  *
2077  * @return
2078  *   0 on success, a negative errno value otherwise and rte_errno is set.
2079  */
2080 static int
2081 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
2082                           const struct rte_flow_item *item,
2083                           const struct rte_flow_attr *attr __rte_unused,
2084                           struct rte_flow_error *error)
2085 {
2086         const struct rte_flow_item_tag *spec = item->spec;
2087         const struct rte_flow_item_tag *mask = item->mask;
2088         const struct rte_flow_item_tag nic_mask = {
2089                 .data = RTE_BE32(UINT32_MAX),
2090                 .index = 0xff,
2091         };
2092         int ret;
2093
2094         if (!mlx5_flow_ext_mreg_supported(dev))
2095                 return rte_flow_error_set(error, ENOTSUP,
2096                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2097                                           "extensive metadata register"
2098                                           " isn't supported");
2099         if (!spec)
2100                 return rte_flow_error_set(error, EINVAL,
2101                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2102                                           item->spec,
2103                                           "data cannot be empty");
2104         if (!mask)
2105                 mask = &rte_flow_item_tag_mask;
2106         if (!mask->data)
2107                 return rte_flow_error_set(error, EINVAL,
2108                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2109                                         "mask cannot be zero");
2110
2111         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2112                                         (const uint8_t *)&nic_mask,
2113                                         sizeof(struct rte_flow_item_tag),
2114                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2115         if (ret < 0)
2116                 return ret;
2117         if (mask->index != 0xff)
2118                 return rte_flow_error_set(error, EINVAL,
2119                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2120                                           "partial mask for tag index"
2121                                           " is not supported");
2122         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
2123         if (ret < 0)
2124                 return ret;
2125         MLX5_ASSERT(ret != REG_NON);
2126         return 0;
2127 }
2128
2129 /**
2130  * Validate vport item.
2131  *
2132  * @param[in] dev
2133  *   Pointer to the rte_eth_dev structure.
2134  * @param[in] item
2135  *   Item specification.
2136  * @param[in] attr
2137  *   Attributes of flow that includes this item.
2138  * @param[in] item_flags
2139  *   Bit-fields that holds the items detected until now.
2140  * @param[out] error
2141  *   Pointer to error structure.
2142  *
2143  * @return
2144  *   0 on success, a negative errno value otherwise and rte_errno is set.
2145  */
2146 static int
2147 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
2148                               const struct rte_flow_item *item,
2149                               const struct rte_flow_attr *attr,
2150                               uint64_t item_flags,
2151                               struct rte_flow_error *error)
2152 {
2153         const struct rte_flow_item_port_id *spec = item->spec;
2154         const struct rte_flow_item_port_id *mask = item->mask;
2155         const struct rte_flow_item_port_id switch_mask = {
2156                         .id = 0xffffffff,
2157         };
2158         struct mlx5_priv *esw_priv;
2159         struct mlx5_priv *dev_priv;
2160         int ret;
2161
2162         if (!attr->transfer)
2163                 return rte_flow_error_set(error, EINVAL,
2164                                           RTE_FLOW_ERROR_TYPE_ITEM,
2165                                           NULL,
2166                                           "match on port id is valid only"
2167                                           " when transfer flag is enabled");
2168         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
2169                 return rte_flow_error_set(error, ENOTSUP,
2170                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2171                                           "multiple source ports are not"
2172                                           " supported");
2173         if (!mask)
2174                 mask = &switch_mask;
2175         if (mask->id != 0xffffffff)
2176                 return rte_flow_error_set(error, ENOTSUP,
2177                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2178                                            mask,
2179                                            "no support for partial mask on"
2180                                            " \"id\" field");
2181         ret = mlx5_flow_item_acceptable
2182                                 (item, (const uint8_t *)mask,
2183                                  (const uint8_t *)&rte_flow_item_port_id_mask,
2184                                  sizeof(struct rte_flow_item_port_id),
2185                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2186         if (ret)
2187                 return ret;
2188         if (!spec)
2189                 return 0;
2190         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
2191         if (!esw_priv)
2192                 return rte_flow_error_set(error, rte_errno,
2193                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2194                                           "failed to obtain E-Switch info for"
2195                                           " port");
2196         dev_priv = mlx5_dev_to_eswitch_info(dev);
2197         if (!dev_priv)
2198                 return rte_flow_error_set(error, rte_errno,
2199                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2200                                           NULL,
2201                                           "failed to obtain E-Switch info");
2202         if (esw_priv->domain_id != dev_priv->domain_id)
2203                 return rte_flow_error_set(error, EINVAL,
2204                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2205                                           "cannot match on a port from a"
2206                                           " different E-Switch");
2207         return 0;
2208 }
2209
2210 /**
2211  * Validate VLAN item.
2212  *
2213  * @param[in] item
2214  *   Item specification.
2215  * @param[in] item_flags
2216  *   Bit-fields that holds the items detected until now.
2217  * @param[in] dev
2218  *   Ethernet device flow is being created on.
2219  * @param[out] error
2220  *   Pointer to error structure.
2221  *
2222  * @return
2223  *   0 on success, a negative errno value otherwise and rte_errno is set.
2224  */
2225 static int
2226 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
2227                            uint64_t item_flags,
2228                            struct rte_eth_dev *dev,
2229                            struct rte_flow_error *error)
2230 {
2231         const struct rte_flow_item_vlan *mask = item->mask;
2232         const struct rte_flow_item_vlan nic_mask = {
2233                 .tci = RTE_BE16(UINT16_MAX),
2234                 .inner_type = RTE_BE16(UINT16_MAX),
2235                 .has_more_vlan = 1,
2236         };
2237         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2238         int ret;
2239         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2240                                         MLX5_FLOW_LAYER_INNER_L4) :
2241                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2242                                         MLX5_FLOW_LAYER_OUTER_L4);
2243         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2244                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2245
2246         if (item_flags & vlanm)
2247                 return rte_flow_error_set(error, EINVAL,
2248                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2249                                           "multiple VLAN layers not supported");
2250         else if ((item_flags & l34m) != 0)
2251                 return rte_flow_error_set(error, EINVAL,
2252                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2253                                           "VLAN cannot follow L3/L4 layer");
2254         if (!mask)
2255                 mask = &rte_flow_item_vlan_mask;
2256         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2257                                         (const uint8_t *)&nic_mask,
2258                                         sizeof(struct rte_flow_item_vlan),
2259                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2260         if (ret)
2261                 return ret;
2262         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2263                 struct mlx5_priv *priv = dev->data->dev_private;
2264
2265                 if (priv->vmwa_context) {
2266                         /*
2267                          * Non-NULL context means we have a virtual machine
2268                          * and SR-IOV enabled, we have to create VLAN interface
2269                          * to make hypervisor to setup E-Switch vport
2270                          * context correctly. We avoid creating the multiple
2271                          * VLAN interfaces, so we cannot support VLAN tag mask.
2272                          */
2273                         return rte_flow_error_set(error, EINVAL,
2274                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2275                                                   item,
2276                                                   "VLAN tag mask is not"
2277                                                   " supported in virtual"
2278                                                   " environment");
2279                 }
2280         }
2281         return 0;
2282 }
2283
2284 /*
2285  * GTP flags are contained in 1 byte of the format:
2286  * -------------------------------------------
2287  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
2288  * |-----------------------------------------|
2289  * | value | Version | PT | Res | E | S | PN |
2290  * -------------------------------------------
2291  *
2292  * Matching is supported only for GTP flags E, S, PN.
2293  */
2294 #define MLX5_GTP_FLAGS_MASK     0x07
2295
2296 /**
2297  * Validate GTP item.
2298  *
2299  * @param[in] dev
2300  *   Pointer to the rte_eth_dev structure.
2301  * @param[in] item
2302  *   Item specification.
2303  * @param[in] item_flags
2304  *   Bit-fields that holds the items detected until now.
2305  * @param[out] error
2306  *   Pointer to error structure.
2307  *
2308  * @return
2309  *   0 on success, a negative errno value otherwise and rte_errno is set.
2310  */
2311 static int
2312 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
2313                           const struct rte_flow_item *item,
2314                           uint64_t item_flags,
2315                           struct rte_flow_error *error)
2316 {
2317         struct mlx5_priv *priv = dev->data->dev_private;
2318         const struct rte_flow_item_gtp *spec = item->spec;
2319         const struct rte_flow_item_gtp *mask = item->mask;
2320         const struct rte_flow_item_gtp nic_mask = {
2321                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
2322                 .msg_type = 0xff,
2323                 .teid = RTE_BE32(0xffffffff),
2324         };
2325
2326         if (!priv->config.hca_attr.tunnel_stateless_gtp)
2327                 return rte_flow_error_set(error, ENOTSUP,
2328                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2329                                           "GTP support is not enabled");
2330         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2331                 return rte_flow_error_set(error, ENOTSUP,
2332                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2333                                           "multiple tunnel layers not"
2334                                           " supported");
2335         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2336                 return rte_flow_error_set(error, EINVAL,
2337                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2338                                           "no outer UDP layer found");
2339         if (!mask)
2340                 mask = &rte_flow_item_gtp_mask;
2341         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
2342                 return rte_flow_error_set(error, ENOTSUP,
2343                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2344                                           "Match is supported for GTP"
2345                                           " flags only");
2346         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2347                                          (const uint8_t *)&nic_mask,
2348                                          sizeof(struct rte_flow_item_gtp),
2349                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2350 }
2351
2352 /**
2353  * Validate GTP PSC item.
2354  *
2355  * @param[in] item
2356  *   Item specification.
2357  * @param[in] last_item
2358  *   Previous validated item in the pattern items.
2359  * @param[in] gtp_item
2360  *   Previous GTP item specification.
2361  * @param[in] attr
2362  *   Pointer to flow attributes.
2363  * @param[out] error
2364  *   Pointer to error structure.
2365  *
2366  * @return
2367  *   0 on success, a negative errno value otherwise and rte_errno is set.
2368  */
2369 static int
2370 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
2371                               uint64_t last_item,
2372                               const struct rte_flow_item *gtp_item,
2373                               const struct rte_flow_attr *attr,
2374                               struct rte_flow_error *error)
2375 {
2376         const struct rte_flow_item_gtp *gtp_spec;
2377         const struct rte_flow_item_gtp *gtp_mask;
2378         const struct rte_flow_item_gtp_psc *spec;
2379         const struct rte_flow_item_gtp_psc *mask;
2380         const struct rte_flow_item_gtp_psc nic_mask = {
2381                 .pdu_type = 0xFF,
2382                 .qfi = 0xFF,
2383         };
2384
2385         if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
2386                 return rte_flow_error_set
2387                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2388                          "GTP PSC item must be preceded with GTP item");
2389         gtp_spec = gtp_item->spec;
2390         gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
2391         /* GTP spec and E flag is requested to match zero. */
2392         if (gtp_spec &&
2393                 (gtp_mask->v_pt_rsv_flags &
2394                 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
2395                 return rte_flow_error_set
2396                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2397                          "GTP E flag must be 1 to match GTP PSC");
2398         /* Check the flow is not created in group zero. */
2399         if (!attr->transfer && !attr->group)
2400                 return rte_flow_error_set
2401                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2402                          "GTP PSC is not supported for group 0");
2403         /* GTP spec is here and E flag is requested to match zero. */
2404         if (!item->spec)
2405                 return 0;
2406         spec = item->spec;
2407         mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
2408         if (spec->pdu_type > MLX5_GTP_EXT_MAX_PDU_TYPE)
2409                 return rte_flow_error_set
2410                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2411                          "PDU type should be smaller than 16");
2412         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2413                                          (const uint8_t *)&nic_mask,
2414                                          sizeof(struct rte_flow_item_gtp_psc),
2415                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2416 }
2417
2418 /**
2419  * Validate IPV4 item.
2420  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
2421  * add specific validation of fragment_offset field,
2422  *
2423  * @param[in] item
2424  *   Item specification.
2425  * @param[in] item_flags
2426  *   Bit-fields that holds the items detected until now.
2427  * @param[out] error
2428  *   Pointer to error structure.
2429  *
2430  * @return
2431  *   0 on success, a negative errno value otherwise and rte_errno is set.
2432  */
2433 static int
2434 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
2435                            uint64_t item_flags,
2436                            uint64_t last_item,
2437                            uint16_t ether_type,
2438                            struct rte_flow_error *error)
2439 {
2440         int ret;
2441         const struct rte_flow_item_ipv4 *spec = item->spec;
2442         const struct rte_flow_item_ipv4 *last = item->last;
2443         const struct rte_flow_item_ipv4 *mask = item->mask;
2444         rte_be16_t fragment_offset_spec = 0;
2445         rte_be16_t fragment_offset_last = 0;
2446         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
2447                 .hdr = {
2448                         .src_addr = RTE_BE32(0xffffffff),
2449                         .dst_addr = RTE_BE32(0xffffffff),
2450                         .type_of_service = 0xff,
2451                         .fragment_offset = RTE_BE16(0xffff),
2452                         .next_proto_id = 0xff,
2453                         .time_to_live = 0xff,
2454                 },
2455         };
2456
2457         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
2458                                            ether_type, &nic_ipv4_mask,
2459                                            MLX5_ITEM_RANGE_ACCEPTED, error);
2460         if (ret < 0)
2461                 return ret;
2462         if (spec && mask)
2463                 fragment_offset_spec = spec->hdr.fragment_offset &
2464                                        mask->hdr.fragment_offset;
2465         if (!fragment_offset_spec)
2466                 return 0;
2467         /*
2468          * spec and mask are valid, enforce using full mask to make sure the
2469          * complete value is used correctly.
2470          */
2471         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2472                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2473                 return rte_flow_error_set(error, EINVAL,
2474                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2475                                           item, "must use full mask for"
2476                                           " fragment_offset");
2477         /*
2478          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
2479          * indicating this is 1st fragment of fragmented packet.
2480          * This is not yet supported in MLX5, return appropriate error message.
2481          */
2482         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
2483                 return rte_flow_error_set(error, ENOTSUP,
2484                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2485                                           "match on first fragment not "
2486                                           "supported");
2487         if (fragment_offset_spec && !last)
2488                 return rte_flow_error_set(error, ENOTSUP,
2489                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2490                                           "specified value not supported");
2491         /* spec and last are valid, validate the specified range. */
2492         fragment_offset_last = last->hdr.fragment_offset &
2493                                mask->hdr.fragment_offset;
2494         /*
2495          * Match on fragment_offset spec 0x2001 and last 0x3fff
2496          * means MF is 1 and frag-offset is > 0.
2497          * This packet is fragment 2nd and onward, excluding last.
2498          * This is not yet supported in MLX5, return appropriate
2499          * error message.
2500          */
2501         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
2502             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2503                 return rte_flow_error_set(error, ENOTSUP,
2504                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2505                                           last, "match on following "
2506                                           "fragments not supported");
2507         /*
2508          * Match on fragment_offset spec 0x0001 and last 0x1fff
2509          * means MF is 0 and frag-offset is > 0.
2510          * This packet is last fragment of fragmented packet.
2511          * This is not yet supported in MLX5, return appropriate
2512          * error message.
2513          */
2514         if (fragment_offset_spec == RTE_BE16(1) &&
2515             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
2516                 return rte_flow_error_set(error, ENOTSUP,
2517                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2518                                           last, "match on last "
2519                                           "fragment not supported");
2520         /*
2521          * Match on fragment_offset spec 0x0001 and last 0x3fff
2522          * means MF and/or frag-offset is not 0.
2523          * This is a fragmented packet.
2524          * Other range values are invalid and rejected.
2525          */
2526         if (!(fragment_offset_spec == RTE_BE16(1) &&
2527               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
2528                 return rte_flow_error_set(error, ENOTSUP,
2529                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2530                                           "specified range not supported");
2531         return 0;
2532 }
2533
2534 /**
2535  * Validate IPV6 fragment extension item.
2536  *
2537  * @param[in] item
2538  *   Item specification.
2539  * @param[in] item_flags
2540  *   Bit-fields that holds the items detected until now.
2541  * @param[out] error
2542  *   Pointer to error structure.
2543  *
2544  * @return
2545  *   0 on success, a negative errno value otherwise and rte_errno is set.
2546  */
2547 static int
2548 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
2549                                     uint64_t item_flags,
2550                                     struct rte_flow_error *error)
2551 {
2552         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
2553         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
2554         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
2555         rte_be16_t frag_data_spec = 0;
2556         rte_be16_t frag_data_last = 0;
2557         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2558         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2559                                       MLX5_FLOW_LAYER_OUTER_L4;
2560         int ret = 0;
2561         struct rte_flow_item_ipv6_frag_ext nic_mask = {
2562                 .hdr = {
2563                         .next_header = 0xff,
2564                         .frag_data = RTE_BE16(0xffff),
2565                 },
2566         };
2567
2568         if (item_flags & l4m)
2569                 return rte_flow_error_set(error, EINVAL,
2570                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2571                                           "ipv6 fragment extension item cannot "
2572                                           "follow L4 item.");
2573         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
2574             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2575                 return rte_flow_error_set(error, EINVAL,
2576                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2577                                           "ipv6 fragment extension item must "
2578                                           "follow ipv6 item");
2579         if (spec && mask)
2580                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
2581         if (!frag_data_spec)
2582                 return 0;
2583         /*
2584          * spec and mask are valid, enforce using full mask to make sure the
2585          * complete value is used correctly.
2586          */
2587         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
2588                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2589                 return rte_flow_error_set(error, EINVAL,
2590                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2591                                           item, "must use full mask for"
2592                                           " frag_data");
2593         /*
2594          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
2595          * This is 1st fragment of fragmented packet.
2596          */
2597         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
2598                 return rte_flow_error_set(error, ENOTSUP,
2599                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2600                                           "match on first fragment not "
2601                                           "supported");
2602         if (frag_data_spec && !last)
2603                 return rte_flow_error_set(error, EINVAL,
2604                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2605                                           "specified value not supported");
2606         ret = mlx5_flow_item_acceptable
2607                                 (item, (const uint8_t *)mask,
2608                                  (const uint8_t *)&nic_mask,
2609                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
2610                                  MLX5_ITEM_RANGE_ACCEPTED, error);
2611         if (ret)
2612                 return ret;
2613         /* spec and last are valid, validate the specified range. */
2614         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
2615         /*
2616          * Match on frag_data spec 0x0009 and last 0xfff9
2617          * means M is 1 and frag-offset is > 0.
2618          * This packet is fragment 2nd and onward, excluding last.
2619          * This is not yet supported in MLX5, return appropriate
2620          * error message.
2621          */
2622         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2623                                        RTE_IPV6_EHDR_MF_MASK) &&
2624             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2625                 return rte_flow_error_set(error, ENOTSUP,
2626                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2627                                           last, "match on following "
2628                                           "fragments not supported");
2629         /*
2630          * Match on frag_data spec 0x0008 and last 0xfff8
2631          * means M is 0 and frag-offset is > 0.
2632          * This packet is last fragment of fragmented packet.
2633          * This is not yet supported in MLX5, return appropriate
2634          * error message.
2635          */
2636         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2637             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2638                 return rte_flow_error_set(error, ENOTSUP,
2639                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2640                                           last, "match on last "
2641                                           "fragment not supported");
2642         /* Other range values are invalid and rejected. */
2643         return rte_flow_error_set(error, EINVAL,
2644                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2645                                   "specified range not supported");
2646 }
2647
2648 /*
2649  * Validate ASO CT item.
2650  *
2651  * @param[in] dev
2652  *   Pointer to the rte_eth_dev structure.
2653  * @param[in] item
2654  *   Item specification.
2655  * @param[in] item_flags
2656  *   Pointer to bit-fields that holds the items detected until now.
2657  * @param[out] error
2658  *   Pointer to error structure.
2659  *
2660  * @return
2661  *   0 on success, a negative errno value otherwise and rte_errno is set.
2662  */
2663 static int
2664 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
2665                              const struct rte_flow_item *item,
2666                              uint64_t *item_flags,
2667                              struct rte_flow_error *error)
2668 {
2669         const struct rte_flow_item_conntrack *spec = item->spec;
2670         const struct rte_flow_item_conntrack *mask = item->mask;
2671         RTE_SET_USED(dev);
2672         uint32_t flags;
2673
2674         if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
2675                 return rte_flow_error_set(error, EINVAL,
2676                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2677                                           "Only one CT is supported");
2678         if (!mask)
2679                 mask = &rte_flow_item_conntrack_mask;
2680         flags = spec->flags & mask->flags;
2681         if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
2682             ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
2683              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
2684              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
2685                 return rte_flow_error_set(error, EINVAL,
2686                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2687                                           "Conflict status bits");
2688         /* State change also needs to be considered. */
2689         *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
2690         return 0;
2691 }
2692
2693 /**
2694  * Validate the pop VLAN action.
2695  *
2696  * @param[in] dev
2697  *   Pointer to the rte_eth_dev structure.
2698  * @param[in] action_flags
2699  *   Holds the actions detected until now.
2700  * @param[in] action
2701  *   Pointer to the pop vlan action.
2702  * @param[in] item_flags
2703  *   The items found in this flow rule.
2704  * @param[in] attr
2705  *   Pointer to flow attributes.
2706  * @param[out] error
2707  *   Pointer to error structure.
2708  *
2709  * @return
2710  *   0 on success, a negative errno value otherwise and rte_errno is set.
2711  */
2712 static int
2713 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2714                                  uint64_t action_flags,
2715                                  const struct rte_flow_action *action,
2716                                  uint64_t item_flags,
2717                                  const struct rte_flow_attr *attr,
2718                                  struct rte_flow_error *error)
2719 {
2720         const struct mlx5_priv *priv = dev->data->dev_private;
2721
2722         (void)action;
2723         (void)attr;
2724         if (!priv->sh->pop_vlan_action)
2725                 return rte_flow_error_set(error, ENOTSUP,
2726                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2727                                           NULL,
2728                                           "pop vlan action is not supported");
2729         if (attr->egress)
2730                 return rte_flow_error_set(error, ENOTSUP,
2731                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2732                                           NULL,
2733                                           "pop vlan action not supported for "
2734                                           "egress");
2735         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2736                 return rte_flow_error_set(error, ENOTSUP,
2737                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2738                                           "no support for multiple VLAN "
2739                                           "actions");
2740         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2741         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2742             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2743                 return rte_flow_error_set(error, ENOTSUP,
2744                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2745                                           NULL,
2746                                           "cannot pop vlan after decap without "
2747                                           "match on inner vlan in the flow");
2748         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2749         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2750             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2751                 return rte_flow_error_set(error, ENOTSUP,
2752                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2753                                           NULL,
2754                                           "cannot pop vlan without a "
2755                                           "match on (outer) vlan in the flow");
2756         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2757                 return rte_flow_error_set(error, EINVAL,
2758                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2759                                           "wrong action order, port_id should "
2760                                           "be after pop VLAN action");
2761         if (!attr->transfer && priv->representor)
2762                 return rte_flow_error_set(error, ENOTSUP,
2763                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2764                                           "pop vlan action for VF representor "
2765                                           "not supported on NIC table");
2766         return 0;
2767 }
2768
2769 /**
2770  * Get VLAN default info from vlan match info.
2771  *
2772  * @param[in] items
2773  *   the list of item specifications.
2774  * @param[out] vlan
2775  *   pointer VLAN info to fill to.
2776  *
2777  * @return
2778  *   0 on success, a negative errno value otherwise and rte_errno is set.
2779  */
2780 static void
2781 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2782                                   struct rte_vlan_hdr *vlan)
2783 {
2784         const struct rte_flow_item_vlan nic_mask = {
2785                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2786                                 MLX5DV_FLOW_VLAN_VID_MASK),
2787                 .inner_type = RTE_BE16(0xffff),
2788         };
2789
2790         if (items == NULL)
2791                 return;
2792         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2793                 int type = items->type;
2794
2795                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2796                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2797                         break;
2798         }
2799         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2800                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2801                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2802
2803                 /* If VLAN item in pattern doesn't contain data, return here. */
2804                 if (!vlan_v)
2805                         return;
2806                 if (!vlan_m)
2807                         vlan_m = &nic_mask;
2808                 /* Only full match values are accepted */
2809                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2810                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2811                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2812                         vlan->vlan_tci |=
2813                                 rte_be_to_cpu_16(vlan_v->tci &
2814                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2815                 }
2816                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2817                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2818                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2819                         vlan->vlan_tci |=
2820                                 rte_be_to_cpu_16(vlan_v->tci &
2821                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2822                 }
2823                 if (vlan_m->inner_type == nic_mask.inner_type)
2824                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2825                                                            vlan_m->inner_type);
2826         }
2827 }
2828
2829 /**
2830  * Validate the push VLAN action.
2831  *
2832  * @param[in] dev
2833  *   Pointer to the rte_eth_dev structure.
2834  * @param[in] action_flags
2835  *   Holds the actions detected until now.
2836  * @param[in] item_flags
2837  *   The items found in this flow rule.
2838  * @param[in] action
2839  *   Pointer to the action structure.
2840  * @param[in] attr
2841  *   Pointer to flow attributes
2842  * @param[out] error
2843  *   Pointer to error structure.
2844  *
2845  * @return
2846  *   0 on success, a negative errno value otherwise and rte_errno is set.
2847  */
2848 static int
2849 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2850                                   uint64_t action_flags,
2851                                   const struct rte_flow_item_vlan *vlan_m,
2852                                   const struct rte_flow_action *action,
2853                                   const struct rte_flow_attr *attr,
2854                                   struct rte_flow_error *error)
2855 {
2856         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2857         const struct mlx5_priv *priv = dev->data->dev_private;
2858
2859         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2860             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2861                 return rte_flow_error_set(error, EINVAL,
2862                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2863                                           "invalid vlan ethertype");
2864         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2865                 return rte_flow_error_set(error, EINVAL,
2866                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2867                                           "wrong action order, port_id should "
2868                                           "be after push VLAN");
2869         if (!attr->transfer && priv->representor)
2870                 return rte_flow_error_set(error, ENOTSUP,
2871                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2872                                           "push vlan action for VF representor "
2873                                           "not supported on NIC table");
2874         if (vlan_m &&
2875             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2876             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2877                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2878             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2879             !(mlx5_flow_find_action
2880                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2881                 return rte_flow_error_set(error, EINVAL,
2882                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2883                                           "not full match mask on VLAN PCP and "
2884                                           "there is no of_set_vlan_pcp action, "
2885                                           "push VLAN action cannot figure out "
2886                                           "PCP value");
2887         if (vlan_m &&
2888             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2889             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2890                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2891             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2892             !(mlx5_flow_find_action
2893                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2894                 return rte_flow_error_set(error, EINVAL,
2895                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2896                                           "not full match mask on VLAN VID and "
2897                                           "there is no of_set_vlan_vid action, "
2898                                           "push VLAN action cannot figure out "
2899                                           "VID value");
2900         (void)attr;
2901         return 0;
2902 }
2903
2904 /**
2905  * Validate the set VLAN PCP.
2906  *
2907  * @param[in] action_flags
2908  *   Holds the actions detected until now.
2909  * @param[in] actions
2910  *   Pointer to the list of actions remaining in the flow rule.
2911  * @param[out] error
2912  *   Pointer to error structure.
2913  *
2914  * @return
2915  *   0 on success, a negative errno value otherwise and rte_errno is set.
2916  */
2917 static int
2918 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2919                                      const struct rte_flow_action actions[],
2920                                      struct rte_flow_error *error)
2921 {
2922         const struct rte_flow_action *action = actions;
2923         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2924
2925         if (conf->vlan_pcp > 7)
2926                 return rte_flow_error_set(error, EINVAL,
2927                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2928                                           "VLAN PCP value is too big");
2929         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2930                 return rte_flow_error_set(error, ENOTSUP,
2931                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2932                                           "set VLAN PCP action must follow "
2933                                           "the push VLAN action");
2934         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2935                 return rte_flow_error_set(error, ENOTSUP,
2936                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2937                                           "Multiple VLAN PCP modification are "
2938                                           "not supported");
2939         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2940                 return rte_flow_error_set(error, EINVAL,
2941                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2942                                           "wrong action order, port_id should "
2943                                           "be after set VLAN PCP");
2944         return 0;
2945 }
2946
2947 /**
2948  * Validate the set VLAN VID.
2949  *
2950  * @param[in] item_flags
2951  *   Holds the items detected in this rule.
2952  * @param[in] action_flags
2953  *   Holds the actions detected until now.
2954  * @param[in] actions
2955  *   Pointer to the list of actions remaining in the flow rule.
2956  * @param[out] error
2957  *   Pointer to error structure.
2958  *
2959  * @return
2960  *   0 on success, a negative errno value otherwise and rte_errno is set.
2961  */
2962 static int
2963 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2964                                      uint64_t action_flags,
2965                                      const struct rte_flow_action actions[],
2966                                      struct rte_flow_error *error)
2967 {
2968         const struct rte_flow_action *action = actions;
2969         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2970
2971         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2972                 return rte_flow_error_set(error, EINVAL,
2973                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2974                                           "VLAN VID value is too big");
2975         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2976             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2977                 return rte_flow_error_set(error, ENOTSUP,
2978                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2979                                           "set VLAN VID action must follow push"
2980                                           " VLAN action or match on VLAN item");
2981         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2982                 return rte_flow_error_set(error, ENOTSUP,
2983                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2984                                           "Multiple VLAN VID modifications are "
2985                                           "not supported");
2986         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2987                 return rte_flow_error_set(error, EINVAL,
2988                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2989                                           "wrong action order, port_id should "
2990                                           "be after set VLAN VID");
2991         return 0;
2992 }
2993
2994 /*
2995  * Validate the FLAG action.
2996  *
2997  * @param[in] dev
2998  *   Pointer to the rte_eth_dev structure.
2999  * @param[in] action_flags
3000  *   Holds the actions detected until now.
3001  * @param[in] attr
3002  *   Pointer to flow attributes
3003  * @param[out] error
3004  *   Pointer to error structure.
3005  *
3006  * @return
3007  *   0 on success, a negative errno value otherwise and rte_errno is set.
3008  */
3009 static int
3010 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
3011                              uint64_t action_flags,
3012                              const struct rte_flow_attr *attr,
3013                              struct rte_flow_error *error)
3014 {
3015         struct mlx5_priv *priv = dev->data->dev_private;
3016         struct mlx5_dev_config *config = &priv->config;
3017         int ret;
3018
3019         /* Fall back if no extended metadata register support. */
3020         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3021                 return mlx5_flow_validate_action_flag(action_flags, attr,
3022                                                       error);
3023         /* Extensive metadata mode requires registers. */
3024         if (!mlx5_flow_ext_mreg_supported(dev))
3025                 return rte_flow_error_set(error, ENOTSUP,
3026                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3027                                           "no metadata registers "
3028                                           "to support flag action");
3029         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
3030                 return rte_flow_error_set(error, ENOTSUP,
3031                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3032                                           "extended metadata register"
3033                                           " isn't available");
3034         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3035         if (ret < 0)
3036                 return ret;
3037         MLX5_ASSERT(ret > 0);
3038         if (action_flags & MLX5_FLOW_ACTION_MARK)
3039                 return rte_flow_error_set(error, EINVAL,
3040                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3041                                           "can't mark and flag in same flow");
3042         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3043                 return rte_flow_error_set(error, EINVAL,
3044                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3045                                           "can't have 2 flag"
3046                                           " actions in same flow");
3047         return 0;
3048 }
3049
3050 /**
3051  * Validate MARK action.
3052  *
3053  * @param[in] dev
3054  *   Pointer to the rte_eth_dev structure.
3055  * @param[in] action
3056  *   Pointer to action.
3057  * @param[in] action_flags
3058  *   Holds the actions detected until now.
3059  * @param[in] attr
3060  *   Pointer to flow attributes
3061  * @param[out] error
3062  *   Pointer to error structure.
3063  *
3064  * @return
3065  *   0 on success, a negative errno value otherwise and rte_errno is set.
3066  */
3067 static int
3068 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
3069                              const struct rte_flow_action *action,
3070                              uint64_t action_flags,
3071                              const struct rte_flow_attr *attr,
3072                              struct rte_flow_error *error)
3073 {
3074         struct mlx5_priv *priv = dev->data->dev_private;
3075         struct mlx5_dev_config *config = &priv->config;
3076         const struct rte_flow_action_mark *mark = action->conf;
3077         int ret;
3078
3079         if (is_tunnel_offload_active(dev))
3080                 return rte_flow_error_set(error, ENOTSUP,
3081                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3082                                           "no mark action "
3083                                           "if tunnel offload active");
3084         /* Fall back if no extended metadata register support. */
3085         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3086                 return mlx5_flow_validate_action_mark(action, action_flags,
3087                                                       attr, error);
3088         /* Extensive metadata mode requires registers. */
3089         if (!mlx5_flow_ext_mreg_supported(dev))
3090                 return rte_flow_error_set(error, ENOTSUP,
3091                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3092                                           "no metadata registers "
3093                                           "to support mark action");
3094         if (!priv->sh->dv_mark_mask)
3095                 return rte_flow_error_set(error, ENOTSUP,
3096                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3097                                           "extended metadata register"
3098                                           " isn't available");
3099         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3100         if (ret < 0)
3101                 return ret;
3102         MLX5_ASSERT(ret > 0);
3103         if (!mark)
3104                 return rte_flow_error_set(error, EINVAL,
3105                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3106                                           "configuration cannot be null");
3107         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
3108                 return rte_flow_error_set(error, EINVAL,
3109                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3110                                           &mark->id,
3111                                           "mark id exceeds the limit");
3112         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3113                 return rte_flow_error_set(error, EINVAL,
3114                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3115                                           "can't flag and mark in same flow");
3116         if (action_flags & MLX5_FLOW_ACTION_MARK)
3117                 return rte_flow_error_set(error, EINVAL,
3118                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3119                                           "can't have 2 mark actions in same"
3120                                           " flow");
3121         return 0;
3122 }
3123
3124 /**
3125  * Validate SET_META action.
3126  *
3127  * @param[in] dev
3128  *   Pointer to the rte_eth_dev structure.
3129  * @param[in] action
3130  *   Pointer to the action structure.
3131  * @param[in] action_flags
3132  *   Holds the actions detected until now.
3133  * @param[in] attr
3134  *   Pointer to flow attributes
3135  * @param[out] error
3136  *   Pointer to error structure.
3137  *
3138  * @return
3139  *   0 on success, a negative errno value otherwise and rte_errno is set.
3140  */
3141 static int
3142 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
3143                                  const struct rte_flow_action *action,
3144                                  uint64_t action_flags __rte_unused,
3145                                  const struct rte_flow_attr *attr,
3146                                  struct rte_flow_error *error)
3147 {
3148         const struct rte_flow_action_set_meta *conf;
3149         uint32_t nic_mask = UINT32_MAX;
3150         int reg;
3151
3152         if (!mlx5_flow_ext_mreg_supported(dev))
3153                 return rte_flow_error_set(error, ENOTSUP,
3154                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3155                                           "extended metadata register"
3156                                           " isn't supported");
3157         reg = flow_dv_get_metadata_reg(dev, attr, error);
3158         if (reg < 0)
3159                 return reg;
3160         if (reg == REG_NON)
3161                 return rte_flow_error_set(error, ENOTSUP,
3162                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3163                                           "unavalable extended metadata register");
3164         if (reg != REG_A && reg != REG_B) {
3165                 struct mlx5_priv *priv = dev->data->dev_private;
3166
3167                 nic_mask = priv->sh->dv_meta_mask;
3168         }
3169         if (!(action->conf))
3170                 return rte_flow_error_set(error, EINVAL,
3171                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3172                                           "configuration cannot be null");
3173         conf = (const struct rte_flow_action_set_meta *)action->conf;
3174         if (!conf->mask)
3175                 return rte_flow_error_set(error, EINVAL,
3176                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3177                                           "zero mask doesn't have any effect");
3178         if (conf->mask & ~nic_mask)
3179                 return rte_flow_error_set(error, EINVAL,
3180                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3181                                           "meta data must be within reg C0");
3182         return 0;
3183 }
3184
3185 /**
3186  * Validate SET_TAG action.
3187  *
3188  * @param[in] dev
3189  *   Pointer to the rte_eth_dev structure.
3190  * @param[in] action
3191  *   Pointer to the action structure.
3192  * @param[in] action_flags
3193  *   Holds the actions detected until now.
3194  * @param[in] attr
3195  *   Pointer to flow attributes
3196  * @param[out] error
3197  *   Pointer to error structure.
3198  *
3199  * @return
3200  *   0 on success, a negative errno value otherwise and rte_errno is set.
3201  */
3202 static int
3203 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
3204                                 const struct rte_flow_action *action,
3205                                 uint64_t action_flags,
3206                                 const struct rte_flow_attr *attr,
3207                                 struct rte_flow_error *error)
3208 {
3209         const struct rte_flow_action_set_tag *conf;
3210         const uint64_t terminal_action_flags =
3211                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
3212                 MLX5_FLOW_ACTION_RSS;
3213         int ret;
3214
3215         if (!mlx5_flow_ext_mreg_supported(dev))
3216                 return rte_flow_error_set(error, ENOTSUP,
3217                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3218                                           "extensive metadata register"
3219                                           " isn't supported");
3220         if (!(action->conf))
3221                 return rte_flow_error_set(error, EINVAL,
3222                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3223                                           "configuration cannot be null");
3224         conf = (const struct rte_flow_action_set_tag *)action->conf;
3225         if (!conf->mask)
3226                 return rte_flow_error_set(error, EINVAL,
3227                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3228                                           "zero mask doesn't have any effect");
3229         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
3230         if (ret < 0)
3231                 return ret;
3232         if (!attr->transfer && attr->ingress &&
3233             (action_flags & terminal_action_flags))
3234                 return rte_flow_error_set(error, EINVAL,
3235                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3236                                           "set_tag has no effect"
3237                                           " with terminal actions");
3238         return 0;
3239 }
3240
3241 /**
3242  * Check if action counter is shared by either old or new mechanism.
3243  *
3244  * @param[in] action
3245  *   Pointer to the action structure.
3246  *
3247  * @return
3248  *   True when counter is shared, false otherwise.
3249  */
3250 static inline bool
3251 is_shared_action_count(const struct rte_flow_action *action)
3252 {
3253         const struct rte_flow_action_count *count =
3254                         (const struct rte_flow_action_count *)action->conf;
3255
3256         if ((int)action->type == MLX5_RTE_FLOW_ACTION_TYPE_COUNT)
3257                 return true;
3258         return !!(count && count->shared);
3259 }
3260
3261 /**
3262  * Validate count action.
3263  *
3264  * @param[in] dev
3265  *   Pointer to rte_eth_dev structure.
3266  * @param[in] shared
3267  *   Indicator if action is shared.
3268  * @param[in] action_flags
3269  *   Holds the actions detected until now.
3270  * @param[out] error
3271  *   Pointer to error structure.
3272  *
3273  * @return
3274  *   0 on success, a negative errno value otherwise and rte_errno is set.
3275  */
3276 static int
3277 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
3278                               uint64_t action_flags,
3279                               struct rte_flow_error *error)
3280 {
3281         struct mlx5_priv *priv = dev->data->dev_private;
3282
3283         if (!priv->config.devx)
3284                 goto notsup_err;
3285         if (action_flags & MLX5_FLOW_ACTION_COUNT)
3286                 return rte_flow_error_set(error, EINVAL,
3287                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3288                                           "duplicate count actions set");
3289         if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
3290             !priv->sh->flow_hit_aso_en)
3291                 return rte_flow_error_set(error, EINVAL,
3292                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3293                                           "old age and shared count combination is not supported");
3294 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
3295         return 0;
3296 #endif
3297 notsup_err:
3298         return rte_flow_error_set
3299                       (error, ENOTSUP,
3300                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3301                        NULL,
3302                        "count action not supported");
3303 }
3304
3305 /**
3306  * Validate the L2 encap action.
3307  *
3308  * @param[in] dev
3309  *   Pointer to the rte_eth_dev structure.
3310  * @param[in] action_flags
3311  *   Holds the actions detected until now.
3312  * @param[in] action
3313  *   Pointer to the action structure.
3314  * @param[in] attr
3315  *   Pointer to flow attributes.
3316  * @param[out] error
3317  *   Pointer to error structure.
3318  *
3319  * @return
3320  *   0 on success, a negative errno value otherwise and rte_errno is set.
3321  */
3322 static int
3323 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
3324                                  uint64_t action_flags,
3325                                  const struct rte_flow_action *action,
3326                                  const struct rte_flow_attr *attr,
3327                                  struct rte_flow_error *error)
3328 {
3329         const struct mlx5_priv *priv = dev->data->dev_private;
3330
3331         if (!(action->conf))
3332                 return rte_flow_error_set(error, EINVAL,
3333                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3334                                           "configuration cannot be null");
3335         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3336                 return rte_flow_error_set(error, EINVAL,
3337                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3338                                           "can only have a single encap action "
3339                                           "in a flow");
3340         if (!attr->transfer && priv->representor)
3341                 return rte_flow_error_set(error, ENOTSUP,
3342                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3343                                           "encap action for VF representor "
3344                                           "not supported on NIC table");
3345         return 0;
3346 }
3347
3348 /**
3349  * Validate a decap action.
3350  *
3351  * @param[in] dev
3352  *   Pointer to the rte_eth_dev structure.
3353  * @param[in] action_flags
3354  *   Holds the actions detected until now.
3355  * @param[in] action
3356  *   Pointer to the action structure.
3357  * @param[in] item_flags
3358  *   Holds the items detected.
3359  * @param[in] attr
3360  *   Pointer to flow attributes
3361  * @param[out] error
3362  *   Pointer to error structure.
3363  *
3364  * @return
3365  *   0 on success, a negative errno value otherwise and rte_errno is set.
3366  */
3367 static int
3368 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
3369                               uint64_t action_flags,
3370                               const struct rte_flow_action *action,
3371                               const uint64_t item_flags,
3372                               const struct rte_flow_attr *attr,
3373                               struct rte_flow_error *error)
3374 {
3375         const struct mlx5_priv *priv = dev->data->dev_private;
3376
3377         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
3378             !priv->config.decap_en)
3379                 return rte_flow_error_set(error, ENOTSUP,
3380                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3381                                           "decap is not enabled");
3382         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
3383                 return rte_flow_error_set(error, ENOTSUP,
3384                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3385                                           action_flags &
3386                                           MLX5_FLOW_ACTION_DECAP ? "can only "
3387                                           "have a single decap action" : "decap "
3388                                           "after encap is not supported");
3389         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
3390                 return rte_flow_error_set(error, EINVAL,
3391                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3392                                           "can't have decap action after"
3393                                           " modify action");
3394         if (attr->egress)
3395                 return rte_flow_error_set(error, ENOTSUP,
3396                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
3397                                           NULL,
3398                                           "decap action not supported for "
3399                                           "egress");
3400         if (!attr->transfer && priv->representor)
3401                 return rte_flow_error_set(error, ENOTSUP,
3402                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3403                                           "decap action for VF representor "
3404                                           "not supported on NIC table");
3405         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
3406             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
3407                 return rte_flow_error_set(error, ENOTSUP,
3408                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3409                                 "VXLAN item should be present for VXLAN decap");
3410         return 0;
3411 }
3412
3413 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
3414
3415 /**
3416  * Validate the raw encap and decap actions.
3417  *
3418  * @param[in] dev
3419  *   Pointer to the rte_eth_dev structure.
3420  * @param[in] decap
3421  *   Pointer to the decap action.
3422  * @param[in] encap
3423  *   Pointer to the encap action.
3424  * @param[in] attr
3425  *   Pointer to flow attributes
3426  * @param[in/out] action_flags
3427  *   Holds the actions detected until now.
3428  * @param[out] actions_n
3429  *   pointer to the number of actions counter.
3430  * @param[in] action
3431  *   Pointer to the action structure.
3432  * @param[in] item_flags
3433  *   Holds the items detected.
3434  * @param[out] error
3435  *   Pointer to error structure.
3436  *
3437  * @return
3438  *   0 on success, a negative errno value otherwise and rte_errno is set.
3439  */
3440 static int
3441 flow_dv_validate_action_raw_encap_decap
3442         (struct rte_eth_dev *dev,
3443          const struct rte_flow_action_raw_decap *decap,
3444          const struct rte_flow_action_raw_encap *encap,
3445          const struct rte_flow_attr *attr, uint64_t *action_flags,
3446          int *actions_n, const struct rte_flow_action *action,
3447          uint64_t item_flags, struct rte_flow_error *error)
3448 {
3449         const struct mlx5_priv *priv = dev->data->dev_private;
3450         int ret;
3451
3452         if (encap && (!encap->size || !encap->data))
3453                 return rte_flow_error_set(error, EINVAL,
3454                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3455                                           "raw encap data cannot be empty");
3456         if (decap && encap) {
3457                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
3458                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3459                         /* L3 encap. */
3460                         decap = NULL;
3461                 else if (encap->size <=
3462                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3463                            decap->size >
3464                            MLX5_ENCAPSULATION_DECISION_SIZE)
3465                         /* L3 decap. */
3466                         encap = NULL;
3467                 else if (encap->size >
3468                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3469                            decap->size >
3470                            MLX5_ENCAPSULATION_DECISION_SIZE)
3471                         /* 2 L2 actions: encap and decap. */
3472                         ;
3473                 else
3474                         return rte_flow_error_set(error,
3475                                 ENOTSUP,
3476                                 RTE_FLOW_ERROR_TYPE_ACTION,
3477                                 NULL, "unsupported too small "
3478                                 "raw decap and too small raw "
3479                                 "encap combination");
3480         }
3481         if (decap) {
3482                 ret = flow_dv_validate_action_decap(dev, *action_flags, action,
3483                                                     item_flags, attr, error);
3484                 if (ret < 0)
3485                         return ret;
3486                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
3487                 ++(*actions_n);
3488         }
3489         if (encap) {
3490                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
3491                         return rte_flow_error_set(error, ENOTSUP,
3492                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3493                                                   NULL,
3494                                                   "small raw encap size");
3495                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
3496                         return rte_flow_error_set(error, EINVAL,
3497                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3498                                                   NULL,
3499                                                   "more than one encap action");
3500                 if (!attr->transfer && priv->representor)
3501                         return rte_flow_error_set
3502                                         (error, ENOTSUP,
3503                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3504                                          "encap action for VF representor "
3505                                          "not supported on NIC table");
3506                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
3507                 ++(*actions_n);
3508         }
3509         return 0;
3510 }
3511
3512 /*
3513  * Validate the ASO CT action.
3514  *
3515  * @param[in] dev
3516  *   Pointer to the rte_eth_dev structure.
3517  * @param[in] action_flags
3518  *   Holds the actions detected until now.
3519  * @param[in] item_flags
3520  *   The items found in this flow rule.
3521  * @param[in] attr
3522  *   Pointer to flow attributes.
3523  * @param[out] error
3524  *   Pointer to error structure.
3525  *
3526  * @return
3527  *   0 on success, a negative errno value otherwise and rte_errno is set.
3528  */
3529 static int
3530 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
3531                                uint64_t action_flags,
3532                                uint64_t item_flags,
3533                                const struct rte_flow_attr *attr,
3534                                struct rte_flow_error *error)
3535 {
3536         RTE_SET_USED(dev);
3537
3538         if (attr->group == 0 && !attr->transfer)
3539                 return rte_flow_error_set(error, ENOTSUP,
3540                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3541                                           NULL,
3542                                           "Only support non-root table");
3543         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
3544                 return rte_flow_error_set(error, ENOTSUP,
3545                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3546                                           "CT cannot follow a fate action");
3547         if ((action_flags & MLX5_FLOW_ACTION_METER) ||
3548             (action_flags & MLX5_FLOW_ACTION_AGE))
3549                 return rte_flow_error_set(error, EINVAL,
3550                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3551                                           "Only one ASO action is supported");
3552         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3553                 return rte_flow_error_set(error, EINVAL,
3554                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3555                                           "Encap cannot exist before CT");
3556         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3557                 return rte_flow_error_set(error, EINVAL,
3558                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3559                                           "Not a outer TCP packet");
3560         return 0;
3561 }
3562
3563 /**
3564  * Match encap_decap resource.
3565  *
3566  * @param list
3567  *   Pointer to the hash list.
3568  * @param entry
3569  *   Pointer to exist resource entry object.
3570  * @param key
3571  *   Key of the new entry.
3572  * @param ctx_cb
3573  *   Pointer to new encap_decap resource.
3574  *
3575  * @return
3576  *   0 on matching, none-zero otherwise.
3577  */
3578 int
3579 flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
3580                              struct mlx5_hlist_entry *entry,
3581                              uint64_t key __rte_unused, void *cb_ctx)
3582 {
3583         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3584         struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
3585         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3586
3587         cache_resource = container_of(entry,
3588                                       struct mlx5_flow_dv_encap_decap_resource,
3589                                       entry);
3590         if (resource->reformat_type == cache_resource->reformat_type &&
3591             resource->ft_type == cache_resource->ft_type &&
3592             resource->flags == cache_resource->flags &&
3593             resource->size == cache_resource->size &&
3594             !memcmp((const void *)resource->buf,
3595                     (const void *)cache_resource->buf,
3596                     resource->size))
3597                 return 0;
3598         return -1;
3599 }
3600
3601 /**
3602  * Allocate encap_decap resource.
3603  *
3604  * @param list
3605  *   Pointer to the hash list.
3606  * @param entry
3607  *   Pointer to exist resource entry object.
3608  * @param ctx_cb
3609  *   Pointer to new encap_decap resource.
3610  *
3611  * @return
3612  *   0 on matching, none-zero otherwise.
3613  */
3614 struct mlx5_hlist_entry *
3615 flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
3616                               uint64_t key __rte_unused,
3617                               void *cb_ctx)
3618 {
3619         struct mlx5_dev_ctx_shared *sh = list->ctx;
3620         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3621         struct mlx5dv_dr_domain *domain;
3622         struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
3623         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3624         uint32_t idx;
3625         int ret;
3626
3627         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3628                 domain = sh->fdb_domain;
3629         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3630                 domain = sh->rx_domain;
3631         else
3632                 domain = sh->tx_domain;
3633         /* Register new encap/decap resource. */
3634         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
3635                                        &idx);
3636         if (!cache_resource) {
3637                 rte_flow_error_set(ctx->error, ENOMEM,
3638                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3639                                    "cannot allocate resource memory");
3640                 return NULL;
3641         }
3642         *cache_resource = *resource;
3643         cache_resource->idx = idx;
3644         ret = mlx5_flow_os_create_flow_action_packet_reformat
3645                                         (sh->ctx, domain, cache_resource,
3646                                          &cache_resource->action);
3647         if (ret) {
3648                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
3649                 rte_flow_error_set(ctx->error, ENOMEM,
3650                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3651                                    NULL, "cannot create action");
3652                 return NULL;
3653         }
3654
3655         return &cache_resource->entry;
3656 }
3657
3658 /**
3659  * Find existing encap/decap resource or create and register a new one.
3660  *
3661  * @param[in, out] dev
3662  *   Pointer to rte_eth_dev structure.
3663  * @param[in, out] resource
3664  *   Pointer to encap/decap resource.
3665  * @parm[in, out] dev_flow
3666  *   Pointer to the dev_flow.
3667  * @param[out] error
3668  *   pointer to error structure.
3669  *
3670  * @return
3671  *   0 on success otherwise -errno and errno is set.
3672  */
3673 static int
3674 flow_dv_encap_decap_resource_register
3675                         (struct rte_eth_dev *dev,
3676                          struct mlx5_flow_dv_encap_decap_resource *resource,
3677                          struct mlx5_flow *dev_flow,
3678                          struct rte_flow_error *error)
3679 {
3680         struct mlx5_priv *priv = dev->data->dev_private;
3681         struct mlx5_dev_ctx_shared *sh = priv->sh;
3682         struct mlx5_hlist_entry *entry;
3683         union {
3684                 struct {
3685                         uint32_t ft_type:8;
3686                         uint32_t refmt_type:8;
3687                         /*
3688                          * Header reformat actions can be shared between
3689                          * non-root tables. One bit to indicate non-root
3690                          * table or not.
3691                          */
3692                         uint32_t is_root:1;
3693                         uint32_t reserve:15;
3694                 };
3695                 uint32_t v32;
3696         } encap_decap_key = {
3697                 {
3698                         .ft_type = resource->ft_type,
3699                         .refmt_type = resource->reformat_type,
3700                         .is_root = !!dev_flow->dv.group,
3701                         .reserve = 0,
3702                 }
3703         };
3704         struct mlx5_flow_cb_ctx ctx = {
3705                 .error = error,
3706                 .data = resource,
3707         };
3708         uint64_t key64;
3709
3710         resource->flags = dev_flow->dv.group ? 0 : 1;
3711         key64 =  __rte_raw_cksum(&encap_decap_key.v32,
3712                                  sizeof(encap_decap_key.v32), 0);
3713         if (resource->reformat_type !=
3714             MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
3715             resource->size)
3716                 key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
3717         entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
3718         if (!entry)
3719                 return -rte_errno;
3720         resource = container_of(entry, typeof(*resource), entry);
3721         dev_flow->dv.encap_decap = resource;
3722         dev_flow->handle->dvh.rix_encap_decap = resource->idx;
3723         return 0;
3724 }
3725
3726 /**
3727  * Find existing table jump resource or create and register a new one.
3728  *
3729  * @param[in, out] dev
3730  *   Pointer to rte_eth_dev structure.
3731  * @param[in, out] tbl
3732  *   Pointer to flow table resource.
3733  * @parm[in, out] dev_flow
3734  *   Pointer to the dev_flow.
3735  * @param[out] error
3736  *   pointer to error structure.
3737  *
3738  * @return
3739  *   0 on success otherwise -errno and errno is set.
3740  */
3741 static int
3742 flow_dv_jump_tbl_resource_register
3743                         (struct rte_eth_dev *dev __rte_unused,
3744                          struct mlx5_flow_tbl_resource *tbl,
3745                          struct mlx5_flow *dev_flow,
3746                          struct rte_flow_error *error __rte_unused)
3747 {
3748         struct mlx5_flow_tbl_data_entry *tbl_data =
3749                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
3750
3751         MLX5_ASSERT(tbl);
3752         MLX5_ASSERT(tbl_data->jump.action);
3753         dev_flow->handle->rix_jump = tbl_data->idx;
3754         dev_flow->dv.jump = &tbl_data->jump;
3755         return 0;
3756 }
3757
3758 int
3759 flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
3760                          struct mlx5_cache_entry *entry, void *cb_ctx)
3761 {
3762         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3763         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3764         struct mlx5_flow_dv_port_id_action_resource *res =
3765                         container_of(entry, typeof(*res), entry);
3766
3767         return ref->port_id != res->port_id;
3768 }
3769
3770 struct mlx5_cache_entry *
3771 flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
3772                           struct mlx5_cache_entry *entry __rte_unused,
3773                           void *cb_ctx)
3774 {
3775         struct mlx5_dev_ctx_shared *sh = list->ctx;
3776         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3777         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3778         struct mlx5_flow_dv_port_id_action_resource *cache;
3779         uint32_t idx;
3780         int ret;
3781
3782         /* Register new port id action resource. */
3783         cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3784         if (!cache) {
3785                 rte_flow_error_set(ctx->error, ENOMEM,
3786                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3787                                    "cannot allocate port_id action cache memory");
3788                 return NULL;
3789         }
3790         *cache = *ref;
3791         ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
3792                                                         ref->port_id,
3793                                                         &cache->action);
3794         if (ret) {
3795                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
3796                 rte_flow_error_set(ctx->error, ENOMEM,
3797                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3798                                    "cannot create action");
3799                 return NULL;
3800         }
3801         cache->idx = idx;
3802         return &cache->entry;
3803 }
3804
3805 /**
3806  * Find existing table port ID resource or create and register a new one.
3807  *
3808  * @param[in, out] dev
3809  *   Pointer to rte_eth_dev structure.
3810  * @param[in, out] resource
3811  *   Pointer to port ID action resource.
3812  * @parm[in, out] dev_flow
3813  *   Pointer to the dev_flow.
3814  * @param[out] error
3815  *   pointer to error structure.
3816  *
3817  * @return
3818  *   0 on success otherwise -errno and errno is set.
3819  */
3820 static int
3821 flow_dv_port_id_action_resource_register
3822                         (struct rte_eth_dev *dev,
3823                          struct mlx5_flow_dv_port_id_action_resource *resource,
3824                          struct mlx5_flow *dev_flow,
3825                          struct rte_flow_error *error)
3826 {
3827         struct mlx5_priv *priv = dev->data->dev_private;
3828         struct mlx5_cache_entry *entry;
3829         struct mlx5_flow_dv_port_id_action_resource *cache;
3830         struct mlx5_flow_cb_ctx ctx = {
3831                 .error = error,
3832                 .data = resource,
3833         };
3834
3835         entry = mlx5_cache_register(&priv->sh->port_id_action_list, &ctx);
3836         if (!entry)
3837                 return -rte_errno;
3838         cache = container_of(entry, typeof(*cache), entry);
3839         dev_flow->dv.port_id_action = cache;
3840         dev_flow->handle->rix_port_id_action = cache->idx;
3841         return 0;
3842 }
3843
3844 int
3845 flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
3846                          struct mlx5_cache_entry *entry, void *cb_ctx)
3847 {
3848         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3849         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3850         struct mlx5_flow_dv_push_vlan_action_resource *res =
3851                         container_of(entry, typeof(*res), entry);
3852
3853         return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
3854 }
3855
3856 struct mlx5_cache_entry *
3857 flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
3858                           struct mlx5_cache_entry *entry __rte_unused,
3859                           void *cb_ctx)
3860 {
3861         struct mlx5_dev_ctx_shared *sh = list->ctx;
3862         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3863         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3864         struct mlx5_flow_dv_push_vlan_action_resource *cache;
3865         struct mlx5dv_dr_domain *domain;
3866         uint32_t idx;
3867         int ret;
3868
3869         /* Register new port id action resource. */
3870         cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3871         if (!cache) {
3872                 rte_flow_error_set(ctx->error, ENOMEM,
3873                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3874                                    "cannot allocate push_vlan action cache memory");
3875                 return NULL;
3876         }
3877         *cache = *ref;
3878         if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3879                 domain = sh->fdb_domain;
3880         else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3881                 domain = sh->rx_domain;
3882         else
3883                 domain = sh->tx_domain;
3884         ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
3885                                                         &cache->action);
3886         if (ret) {
3887                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
3888                 rte_flow_error_set(ctx->error, ENOMEM,
3889                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3890                                    "cannot create push vlan action");
3891                 return NULL;
3892         }
3893         cache->idx = idx;
3894         return &cache->entry;
3895 }
3896
3897 /**
3898  * Find existing push vlan resource or create and register a new one.
3899  *
3900  * @param [in, out] dev
3901  *   Pointer to rte_eth_dev structure.
3902  * @param[in, out] resource
3903  *   Pointer to port ID action resource.
3904  * @parm[in, out] dev_flow
3905  *   Pointer to the dev_flow.
3906  * @param[out] error
3907  *   pointer to error structure.
3908  *
3909  * @return
3910  *   0 on success otherwise -errno and errno is set.
3911  */
3912 static int
3913 flow_dv_push_vlan_action_resource_register
3914                        (struct rte_eth_dev *dev,
3915                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
3916                         struct mlx5_flow *dev_flow,
3917                         struct rte_flow_error *error)
3918 {
3919         struct mlx5_priv *priv = dev->data->dev_private;
3920         struct mlx5_flow_dv_push_vlan_action_resource *cache;
3921         struct mlx5_cache_entry *entry;
3922         struct mlx5_flow_cb_ctx ctx = {
3923                 .error = error,
3924                 .data = resource,
3925         };
3926
3927         entry = mlx5_cache_register(&priv->sh->push_vlan_action_list, &ctx);
3928         if (!entry)
3929                 return -rte_errno;
3930         cache = container_of(entry, typeof(*cache), entry);
3931
3932         dev_flow->handle->dvh.rix_push_vlan = cache->idx;
3933         dev_flow->dv.push_vlan_res = cache;
3934         return 0;
3935 }
3936
3937 /**
3938  * Get the size of specific rte_flow_item_type hdr size
3939  *
3940  * @param[in] item_type
3941  *   Tested rte_flow_item_type.
3942  *
3943  * @return
3944  *   sizeof struct item_type, 0 if void or irrelevant.
3945  */
3946 static size_t
3947 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3948 {
3949         size_t retval;
3950
3951         switch (item_type) {
3952         case RTE_FLOW_ITEM_TYPE_ETH:
3953                 retval = sizeof(struct rte_ether_hdr);
3954                 break;
3955         case RTE_FLOW_ITEM_TYPE_VLAN:
3956                 retval = sizeof(struct rte_vlan_hdr);
3957                 break;
3958         case RTE_FLOW_ITEM_TYPE_IPV4:
3959                 retval = sizeof(struct rte_ipv4_hdr);
3960                 break;
3961         case RTE_FLOW_ITEM_TYPE_IPV6:
3962                 retval = sizeof(struct rte_ipv6_hdr);
3963                 break;
3964         case RTE_FLOW_ITEM_TYPE_UDP:
3965                 retval = sizeof(struct rte_udp_hdr);
3966                 break;
3967         case RTE_FLOW_ITEM_TYPE_TCP:
3968                 retval = sizeof(struct rte_tcp_hdr);
3969                 break;
3970         case RTE_FLOW_ITEM_TYPE_VXLAN:
3971         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3972                 retval = sizeof(struct rte_vxlan_hdr);
3973                 break;
3974         case RTE_FLOW_ITEM_TYPE_GRE:
3975         case RTE_FLOW_ITEM_TYPE_NVGRE:
3976                 retval = sizeof(struct rte_gre_hdr);
3977                 break;
3978         case RTE_FLOW_ITEM_TYPE_MPLS:
3979                 retval = sizeof(struct rte_mpls_hdr);
3980                 break;
3981         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
3982         default:
3983                 retval = 0;
3984                 break;
3985         }
3986         return retval;
3987 }
3988
3989 #define MLX5_ENCAP_IPV4_VERSION         0x40
3990 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
3991 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
3992 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
3993 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
3994 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
3995 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
3996
3997 /**
3998  * Convert the encap action data from list of rte_flow_item to raw buffer
3999  *
4000  * @param[in] items
4001  *   Pointer to rte_flow_item objects list.
4002  * @param[out] buf
4003  *   Pointer to the output buffer.
4004  * @param[out] size
4005  *   Pointer to the output buffer size.
4006  * @param[out] error
4007  *   Pointer to the error structure.
4008  *
4009  * @return
4010  *   0 on success, a negative errno value otherwise and rte_errno is set.
4011  */
4012 static int
4013 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
4014                            size_t *size, struct rte_flow_error *error)
4015 {
4016         struct rte_ether_hdr *eth = NULL;
4017         struct rte_vlan_hdr *vlan = NULL;
4018         struct rte_ipv4_hdr *ipv4 = NULL;
4019         struct rte_ipv6_hdr *ipv6 = NULL;
4020         struct rte_udp_hdr *udp = NULL;
4021         struct rte_vxlan_hdr *vxlan = NULL;
4022         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
4023         struct rte_gre_hdr *gre = NULL;
4024         size_t len;
4025         size_t temp_size = 0;
4026
4027         if (!items)
4028                 return rte_flow_error_set(error, EINVAL,
4029                                           RTE_FLOW_ERROR_TYPE_ACTION,
4030                                           NULL, "invalid empty data");
4031         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4032                 len = flow_dv_get_item_hdr_len(items->type);
4033                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
4034                         return rte_flow_error_set(error, EINVAL,
4035                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4036                                                   (void *)items->type,
4037                                                   "items total size is too big"
4038                                                   " for encap action");
4039                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
4040                 switch (items->type) {
4041                 case RTE_FLOW_ITEM_TYPE_ETH:
4042                         eth = (struct rte_ether_hdr *)&buf[temp_size];
4043                         break;
4044                 case RTE_FLOW_ITEM_TYPE_VLAN:
4045                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
4046                         if (!eth)
4047                                 return rte_flow_error_set(error, EINVAL,
4048                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4049                                                 (void *)items->type,
4050                                                 "eth header not found");
4051                         if (!eth->ether_type)
4052                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
4053                         break;
4054                 case RTE_FLOW_ITEM_TYPE_IPV4:
4055                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
4056                         if (!vlan && !eth)
4057                                 return rte_flow_error_set(error, EINVAL,
4058                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4059                                                 (void *)items->type,
4060                                                 "neither eth nor vlan"
4061                                                 " header found");
4062                         if (vlan && !vlan->eth_proto)
4063                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4064                         else if (eth && !eth->ether_type)
4065                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4066                         if (!ipv4->version_ihl)
4067                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
4068                                                     MLX5_ENCAP_IPV4_IHL_MIN;
4069                         if (!ipv4->time_to_live)
4070                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
4071                         break;
4072                 case RTE_FLOW_ITEM_TYPE_IPV6:
4073                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
4074                         if (!vlan && !eth)
4075                                 return rte_flow_error_set(error, EINVAL,
4076                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4077                                                 (void *)items->type,
4078                                                 "neither eth nor vlan"
4079                                                 " header found");
4080                         if (vlan && !vlan->eth_proto)
4081                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4082                         else if (eth && !eth->ether_type)
4083                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4084                         if (!ipv6->vtc_flow)
4085                                 ipv6->vtc_flow =
4086                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
4087                         if (!ipv6->hop_limits)
4088                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
4089                         break;
4090                 case RTE_FLOW_ITEM_TYPE_UDP:
4091                         udp = (struct rte_udp_hdr *)&buf[temp_size];
4092                         if (!ipv4 && !ipv6)
4093                                 return rte_flow_error_set(error, EINVAL,
4094                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4095                                                 (void *)items->type,
4096                                                 "ip header not found");
4097                         if (ipv4 && !ipv4->next_proto_id)
4098                                 ipv4->next_proto_id = IPPROTO_UDP;
4099                         else if (ipv6 && !ipv6->proto)
4100                                 ipv6->proto = IPPROTO_UDP;
4101                         break;
4102                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4103                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
4104                         if (!udp)
4105                                 return rte_flow_error_set(error, EINVAL,
4106                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4107                                                 (void *)items->type,
4108                                                 "udp header not found");
4109                         if (!udp->dst_port)
4110                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
4111                         if (!vxlan->vx_flags)
4112                                 vxlan->vx_flags =
4113                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
4114                         break;
4115                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4116                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
4117                         if (!udp)
4118                                 return rte_flow_error_set(error, EINVAL,
4119                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4120                                                 (void *)items->type,
4121                                                 "udp header not found");
4122                         if (!vxlan_gpe->proto)
4123                                 return rte_flow_error_set(error, EINVAL,
4124                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4125                                                 (void *)items->type,
4126                                                 "next protocol not found");
4127                         if (!udp->dst_port)
4128                                 udp->dst_port =
4129                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
4130                         if (!vxlan_gpe->vx_flags)
4131                                 vxlan_gpe->vx_flags =
4132                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
4133                         break;
4134                 case RTE_FLOW_ITEM_TYPE_GRE:
4135                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4136                         gre = (struct rte_gre_hdr *)&buf[temp_size];
4137                         if (!gre->proto)
4138                                 return rte_flow_error_set(error, EINVAL,
4139                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4140                                                 (void *)items->type,
4141                                                 "next protocol not found");
4142                         if (!ipv4 && !ipv6)
4143                                 return rte_flow_error_set(error, EINVAL,
4144                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4145                                                 (void *)items->type,
4146                                                 "ip header not found");
4147                         if (ipv4 && !ipv4->next_proto_id)
4148                                 ipv4->next_proto_id = IPPROTO_GRE;
4149                         else if (ipv6 && !ipv6->proto)
4150                                 ipv6->proto = IPPROTO_GRE;
4151                         break;
4152                 case RTE_FLOW_ITEM_TYPE_VOID:
4153                         break;
4154                 default:
4155                         return rte_flow_error_set(error, EINVAL,
4156                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4157                                                   (void *)items->type,
4158                                                   "unsupported item type");
4159                         break;
4160                 }
4161                 temp_size += len;
4162         }
4163         *size = temp_size;
4164         return 0;
4165 }
4166
4167 static int
4168 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
4169 {
4170         struct rte_ether_hdr *eth = NULL;
4171         struct rte_vlan_hdr *vlan = NULL;
4172         struct rte_ipv6_hdr *ipv6 = NULL;
4173         struct rte_udp_hdr *udp = NULL;
4174         char *next_hdr;
4175         uint16_t proto;
4176
4177         eth = (struct rte_ether_hdr *)data;
4178         next_hdr = (char *)(eth + 1);
4179         proto = RTE_BE16(eth->ether_type);
4180
4181         /* VLAN skipping */
4182         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
4183                 vlan = (struct rte_vlan_hdr *)next_hdr;
4184                 proto = RTE_BE16(vlan->eth_proto);
4185                 next_hdr += sizeof(struct rte_vlan_hdr);
4186         }
4187
4188         /* HW calculates IPv4 csum. no need to proceed */
4189         if (proto == RTE_ETHER_TYPE_IPV4)
4190                 return 0;
4191
4192         /* non IPv4/IPv6 header. not supported */
4193         if (proto != RTE_ETHER_TYPE_IPV6) {
4194                 return rte_flow_error_set(error, ENOTSUP,
4195                                           RTE_FLOW_ERROR_TYPE_ACTION,
4196                                           NULL, "Cannot offload non IPv4/IPv6");
4197         }
4198
4199         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
4200
4201         /* ignore non UDP */
4202         if (ipv6->proto != IPPROTO_UDP)
4203                 return 0;
4204
4205         udp = (struct rte_udp_hdr *)(ipv6 + 1);
4206         udp->dgram_cksum = 0;
4207
4208         return 0;
4209 }
4210
4211 /**
4212  * Convert L2 encap action to DV specification.
4213  *
4214  * @param[in] dev
4215  *   Pointer to rte_eth_dev structure.
4216  * @param[in] action
4217  *   Pointer to action structure.
4218  * @param[in, out] dev_flow
4219  *   Pointer to the mlx5_flow.
4220  * @param[in] transfer
4221  *   Mark if the flow is E-Switch flow.
4222  * @param[out] error
4223  *   Pointer to the error structure.
4224  *
4225  * @return
4226  *   0 on success, a negative errno value otherwise and rte_errno is set.
4227  */
4228 static int
4229 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
4230                                const struct rte_flow_action *action,
4231                                struct mlx5_flow *dev_flow,
4232                                uint8_t transfer,
4233                                struct rte_flow_error *error)
4234 {
4235         const struct rte_flow_item *encap_data;
4236         const struct rte_flow_action_raw_encap *raw_encap_data;
4237         struct mlx5_flow_dv_encap_decap_resource res = {
4238                 .reformat_type =
4239                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
4240                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4241                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
4242         };
4243
4244         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4245                 raw_encap_data =
4246                         (const struct rte_flow_action_raw_encap *)action->conf;
4247                 res.size = raw_encap_data->size;
4248                 memcpy(res.buf, raw_encap_data->data, res.size);
4249         } else {
4250                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
4251                         encap_data =
4252                                 ((const struct rte_flow_action_vxlan_encap *)
4253                                                 action->conf)->definition;
4254                 else
4255                         encap_data =
4256                                 ((const struct rte_flow_action_nvgre_encap *)
4257                                                 action->conf)->definition;
4258                 if (flow_dv_convert_encap_data(encap_data, res.buf,
4259                                                &res.size, error))
4260                         return -rte_errno;
4261         }
4262         if (flow_dv_zero_encap_udp_csum(res.buf, error))
4263                 return -rte_errno;
4264         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4265                 return rte_flow_error_set(error, EINVAL,
4266                                           RTE_FLOW_ERROR_TYPE_ACTION,
4267                                           NULL, "can't create L2 encap action");
4268         return 0;
4269 }
4270
4271 /**
4272  * Convert L2 decap action to DV specification.
4273  *
4274  * @param[in] dev
4275  *   Pointer to rte_eth_dev structure.
4276  * @param[in, out] dev_flow
4277  *   Pointer to the mlx5_flow.
4278  * @param[in] transfer
4279  *   Mark if the flow is E-Switch flow.
4280  * @param[out] error
4281  *   Pointer to the error structure.
4282  *
4283  * @return
4284  *   0 on success, a negative errno value otherwise and rte_errno is set.
4285  */
4286 static int
4287 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
4288                                struct mlx5_flow *dev_flow,
4289                                uint8_t transfer,
4290                                struct rte_flow_error *error)
4291 {
4292         struct mlx5_flow_dv_encap_decap_resource res = {
4293                 .size = 0,
4294                 .reformat_type =
4295                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
4296                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4297                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
4298         };
4299
4300         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4301                 return rte_flow_error_set(error, EINVAL,
4302                                           RTE_FLOW_ERROR_TYPE_ACTION,
4303                                           NULL, "can't create L2 decap action");
4304         return 0;
4305 }
4306
4307 /**
4308  * Convert raw decap/encap (L3 tunnel) action to DV specification.
4309  *
4310  * @param[in] dev
4311  *   Pointer to rte_eth_dev structure.
4312  * @param[in] action
4313  *   Pointer to action structure.
4314  * @param[in, out] dev_flow
4315  *   Pointer to the mlx5_flow.
4316  * @param[in] attr
4317  *   Pointer to the flow attributes.
4318  * @param[out] error
4319  *   Pointer to the error structure.
4320  *
4321  * @return
4322  *   0 on success, a negative errno value otherwise and rte_errno is set.
4323  */
4324 static int
4325 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
4326                                 const struct rte_flow_action *action,
4327                                 struct mlx5_flow *dev_flow,
4328                                 const struct rte_flow_attr *attr,
4329                                 struct rte_flow_error *error)
4330 {
4331         const struct rte_flow_action_raw_encap *encap_data;
4332         struct mlx5_flow_dv_encap_decap_resource res;
4333
4334         memset(&res, 0, sizeof(res));
4335         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
4336         res.size = encap_data->size;
4337         memcpy(res.buf, encap_data->data, res.size);
4338         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
4339                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
4340                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
4341         if (attr->transfer)
4342                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4343         else
4344                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4345                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4346         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4347                 return rte_flow_error_set(error, EINVAL,
4348                                           RTE_FLOW_ERROR_TYPE_ACTION,
4349                                           NULL, "can't create encap action");
4350         return 0;
4351 }
4352
4353 /**
4354  * Create action push VLAN.
4355  *
4356  * @param[in] dev
4357  *   Pointer to rte_eth_dev structure.
4358  * @param[in] attr
4359  *   Pointer to the flow attributes.
4360  * @param[in] vlan
4361  *   Pointer to the vlan to push to the Ethernet header.
4362  * @param[in, out] dev_flow
4363  *   Pointer to the mlx5_flow.
4364  * @param[out] error
4365  *   Pointer to the error structure.
4366  *
4367  * @return
4368  *   0 on success, a negative errno value otherwise and rte_errno is set.
4369  */
4370 static int
4371 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
4372                                 const struct rte_flow_attr *attr,
4373                                 const struct rte_vlan_hdr *vlan,
4374                                 struct mlx5_flow *dev_flow,
4375                                 struct rte_flow_error *error)
4376 {
4377         struct mlx5_flow_dv_push_vlan_action_resource res;
4378
4379         memset(&res, 0, sizeof(res));
4380         res.vlan_tag =
4381                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
4382                                  vlan->vlan_tci);
4383         if (attr->transfer)
4384                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4385         else
4386                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4387                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4388         return flow_dv_push_vlan_action_resource_register
4389                                             (dev, &res, dev_flow, error);
4390 }
4391
4392 /**
4393  * Validate the modify-header actions.
4394  *
4395  * @param[in] action_flags
4396  *   Holds the actions detected until now.
4397  * @param[in] action
4398  *   Pointer to the modify action.
4399  * @param[out] error
4400  *   Pointer to error structure.
4401  *
4402  * @return
4403  *   0 on success, a negative errno value otherwise and rte_errno is set.
4404  */
4405 static int
4406 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
4407                                    const struct rte_flow_action *action,
4408                                    struct rte_flow_error *error)
4409 {
4410         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
4411                 return rte_flow_error_set(error, EINVAL,
4412                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4413                                           NULL, "action configuration not set");
4414         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
4415                 return rte_flow_error_set(error, EINVAL,
4416                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4417                                           "can't have encap action before"
4418                                           " modify action");
4419         return 0;
4420 }
4421
4422 /**
4423  * Validate the modify-header MAC address actions.
4424  *
4425  * @param[in] action_flags
4426  *   Holds the actions detected until now.
4427  * @param[in] action
4428  *   Pointer to the modify action.
4429  * @param[in] item_flags
4430  *   Holds the items detected.
4431  * @param[out] error
4432  *   Pointer to error structure.
4433  *
4434  * @return
4435  *   0 on success, a negative errno value otherwise and rte_errno is set.
4436  */
4437 static int
4438 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
4439                                    const struct rte_flow_action *action,
4440                                    const uint64_t item_flags,
4441                                    struct rte_flow_error *error)
4442 {
4443         int ret = 0;
4444
4445         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4446         if (!ret) {
4447                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
4448                         return rte_flow_error_set(error, EINVAL,
4449                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4450                                                   NULL,
4451                                                   "no L2 item in pattern");
4452         }
4453         return ret;
4454 }
4455
4456 /**
4457  * Validate the modify-header IPv4 address actions.
4458  *
4459  * @param[in] action_flags
4460  *   Holds the actions detected until now.
4461  * @param[in] action
4462  *   Pointer to the modify action.
4463  * @param[in] item_flags
4464  *   Holds the items detected.
4465  * @param[out] error
4466  *   Pointer to error structure.
4467  *
4468  * @return
4469  *   0 on success, a negative errno value otherwise and rte_errno is set.
4470  */
4471 static int
4472 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
4473                                     const struct rte_flow_action *action,
4474                                     const uint64_t item_flags,
4475                                     struct rte_flow_error *error)
4476 {
4477         int ret = 0;
4478         uint64_t layer;
4479
4480         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4481         if (!ret) {
4482                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4483                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4484                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4485                 if (!(item_flags & layer))
4486                         return rte_flow_error_set(error, EINVAL,
4487                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4488                                                   NULL,
4489                                                   "no ipv4 item in pattern");
4490         }
4491         return ret;
4492 }
4493
4494 /**
4495  * Validate the modify-header IPv6 address actions.
4496  *
4497  * @param[in] action_flags
4498  *   Holds the actions detected until now.
4499  * @param[in] action
4500  *   Pointer to the modify action.
4501  * @param[in] item_flags
4502  *   Holds the items detected.
4503  * @param[out] error
4504  *   Pointer to error structure.
4505  *
4506  * @return
4507  *   0 on success, a negative errno value otherwise and rte_errno is set.
4508  */
4509 static int
4510 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
4511                                     const struct rte_flow_action *action,
4512                                     const uint64_t item_flags,
4513                                     struct rte_flow_error *error)
4514 {
4515         int ret = 0;
4516         uint64_t layer;
4517
4518         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4519         if (!ret) {
4520                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4521                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4522                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4523                 if (!(item_flags & layer))
4524                         return rte_flow_error_set(error, EINVAL,
4525                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4526                                                   NULL,
4527                                                   "no ipv6 item in pattern");
4528         }
4529         return ret;
4530 }
4531
4532 /**
4533  * Validate the modify-header TP actions.
4534  *
4535  * @param[in] action_flags
4536  *   Holds the actions detected until now.
4537  * @param[in] action
4538  *   Pointer to the modify action.
4539  * @param[in] item_flags
4540  *   Holds the items detected.
4541  * @param[out] error
4542  *   Pointer to error structure.
4543  *
4544  * @return
4545  *   0 on success, a negative errno value otherwise and rte_errno is set.
4546  */
4547 static int
4548 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
4549                                   const struct rte_flow_action *action,
4550                                   const uint64_t item_flags,
4551                                   struct rte_flow_error *error)
4552 {
4553         int ret = 0;
4554         uint64_t layer;
4555
4556         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4557         if (!ret) {
4558                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4559                                  MLX5_FLOW_LAYER_INNER_L4 :
4560                                  MLX5_FLOW_LAYER_OUTER_L4;
4561                 if (!(item_flags & layer))
4562                         return rte_flow_error_set(error, EINVAL,
4563                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4564                                                   NULL, "no transport layer "
4565                                                   "in pattern");
4566         }
4567         return ret;
4568 }
4569
4570 /**
4571  * Validate the modify-header actions of increment/decrement
4572  * TCP Sequence-number.
4573  *
4574  * @param[in] action_flags
4575  *   Holds the actions detected until now.
4576  * @param[in] action
4577  *   Pointer to the modify action.
4578  * @param[in] item_flags
4579  *   Holds the items detected.
4580  * @param[out] error
4581  *   Pointer to error structure.
4582  *
4583  * @return
4584  *   0 on success, a negative errno value otherwise and rte_errno is set.
4585  */
4586 static int
4587 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
4588                                        const struct rte_flow_action *action,
4589                                        const uint64_t item_flags,
4590                                        struct rte_flow_error *error)
4591 {
4592         int ret = 0;
4593         uint64_t layer;
4594
4595         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4596         if (!ret) {
4597                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4598                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4599                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4600                 if (!(item_flags & layer))
4601                         return rte_flow_error_set(error, EINVAL,
4602                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4603                                                   NULL, "no TCP item in"
4604                                                   " pattern");
4605                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
4606                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
4607                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
4608                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
4609                         return rte_flow_error_set(error, EINVAL,
4610                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4611                                                   NULL,
4612                                                   "cannot decrease and increase"
4613                                                   " TCP sequence number"
4614                                                   " at the same time");
4615         }
4616         return ret;
4617 }
4618
4619 /**
4620  * Validate the modify-header actions of increment/decrement
4621  * TCP Acknowledgment number.
4622  *
4623  * @param[in] action_flags
4624  *   Holds the actions detected until now.
4625  * @param[in] action
4626  *   Pointer to the modify action.
4627  * @param[in] item_flags
4628  *   Holds the items detected.
4629  * @param[out] error
4630  *   Pointer to error structure.
4631  *
4632  * @return
4633  *   0 on success, a negative errno value otherwise and rte_errno is set.
4634  */
4635 static int
4636 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
4637                                        const struct rte_flow_action *action,
4638                                        const uint64_t item_flags,
4639                                        struct rte_flow_error *error)
4640 {
4641         int ret = 0;
4642         uint64_t layer;
4643
4644         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4645         if (!ret) {
4646                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4647                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4648                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4649                 if (!(item_flags & layer))
4650                         return rte_flow_error_set(error, EINVAL,
4651                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4652                                                   NULL, "no TCP item in"
4653                                                   " pattern");
4654                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
4655                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
4656                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
4657                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
4658                         return rte_flow_error_set(error, EINVAL,
4659                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4660                                                   NULL,
4661                                                   "cannot decrease and increase"
4662                                                   " TCP acknowledgment number"
4663                                                   " at the same time");
4664         }
4665         return ret;
4666 }
4667
4668 /**
4669  * Validate the modify-header TTL actions.
4670  *
4671  * @param[in] action_flags
4672  *   Holds the actions detected until now.
4673  * @param[in] action
4674  *   Pointer to the modify action.
4675  * @param[in] item_flags
4676  *   Holds the items detected.
4677  * @param[out] error
4678  *   Pointer to error structure.
4679  *
4680  * @return
4681  *   0 on success, a negative errno value otherwise and rte_errno is set.
4682  */
4683 static int
4684 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
4685                                    const struct rte_flow_action *action,
4686                                    const uint64_t item_flags,
4687                                    struct rte_flow_error *error)
4688 {
4689         int ret = 0;
4690         uint64_t layer;
4691
4692         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4693         if (!ret) {
4694                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4695                                  MLX5_FLOW_LAYER_INNER_L3 :
4696                                  MLX5_FLOW_LAYER_OUTER_L3;
4697                 if (!(item_flags & layer))
4698                         return rte_flow_error_set(error, EINVAL,
4699                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4700                                                   NULL,
4701                                                   "no IP protocol in pattern");
4702         }
4703         return ret;
4704 }
4705
4706 /**
4707  * Validate the generic modify field actions.
4708  * @param[in] dev
4709  *   Pointer to the rte_eth_dev structure.
4710  * @param[in] action_flags
4711  *   Holds the actions detected until now.
4712  * @param[in] action
4713  *   Pointer to the modify action.
4714  * @param[in] attr
4715  *   Pointer to the flow attributes.
4716  * @param[out] error
4717  *   Pointer to error structure.
4718  *
4719  * @return
4720  *   Number of header fields to modify (0 or more) on success,
4721  *   a negative errno value otherwise and rte_errno is set.
4722  */
4723 static int
4724 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev,
4725                                    const uint64_t action_flags,
4726                                    const struct rte_flow_action *action,
4727                                    const struct rte_flow_attr *attr,
4728                                    struct rte_flow_error *error)
4729 {
4730         int ret = 0;
4731         struct mlx5_priv *priv = dev->data->dev_private;
4732         struct mlx5_dev_config *config = &priv->config;
4733         const struct rte_flow_action_modify_field *action_modify_field =
4734                 action->conf;
4735         uint32_t dst_width = mlx5_flow_item_field_width(config,
4736                                 action_modify_field->dst.field);
4737         uint32_t src_width = mlx5_flow_item_field_width(config,
4738                                 action_modify_field->src.field);
4739
4740         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4741         if (ret)
4742                 return ret;
4743
4744         if (action_modify_field->width == 0)
4745                 return rte_flow_error_set(error, EINVAL,
4746                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4747                                 "no bits are requested to be modified");
4748         else if (action_modify_field->width > dst_width ||
4749                  action_modify_field->width > src_width)
4750                 return rte_flow_error_set(error, EINVAL,
4751                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4752                                 "cannot modify more bits than"
4753                                 " the width of a field");
4754         if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE &&
4755             action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) {
4756                 if ((action_modify_field->dst.offset +
4757                      action_modify_field->width > dst_width) ||
4758                     (action_modify_field->dst.offset % 32))
4759                         return rte_flow_error_set(error, EINVAL,
4760                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4761                                         "destination offset is too big"
4762                                         " or not aligned to 4 bytes");
4763                 if (action_modify_field->dst.level &&
4764                     action_modify_field->dst.field != RTE_FLOW_FIELD_TAG)
4765                         return rte_flow_error_set(error, ENOTSUP,
4766                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4767                                         "inner header fields modification"
4768                                         " is not supported");
4769         }
4770         if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE &&
4771             action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) {
4772                 if (!attr->transfer && !attr->group)
4773                         return rte_flow_error_set(error, ENOTSUP,
4774                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4775                                         "modify field action is not"
4776                                         " supported for group 0");
4777                 if ((action_modify_field->src.offset +
4778                      action_modify_field->width > src_width) ||
4779                     (action_modify_field->src.offset % 32))
4780                         return rte_flow_error_set(error, EINVAL,
4781                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4782                                         "source offset is too big"
4783                                         " or not aligned to 4 bytes");
4784                 if (action_modify_field->src.level &&
4785                     action_modify_field->src.field != RTE_FLOW_FIELD_TAG)
4786                         return rte_flow_error_set(error, ENOTSUP,
4787                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4788                                         "inner header fields modification"
4789                                         " is not supported");
4790         }
4791         if (action_modify_field->dst.field ==
4792             action_modify_field->src.field)
4793                 return rte_flow_error_set(error, EINVAL,
4794                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4795                                 "source and destination fields"
4796                                 " cannot be the same");
4797         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE ||
4798             action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER)
4799                 return rte_flow_error_set(error, EINVAL,
4800                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4801                                 "immediate value or a pointer to it"
4802                                 " cannot be used as a destination");
4803         if (action_modify_field->dst.field == RTE_FLOW_FIELD_START ||
4804             action_modify_field->src.field == RTE_FLOW_FIELD_START)
4805                 return rte_flow_error_set(error, ENOTSUP,
4806                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4807                                 "modifications of an arbitrary"
4808                                 " place in a packet is not supported");
4809         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE ||
4810             action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE)
4811                 return rte_flow_error_set(error, ENOTSUP,
4812                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4813                                 "modifications of the 802.1Q Tag"
4814                                 " Identifier is not supported");
4815         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI ||
4816             action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI)
4817                 return rte_flow_error_set(error, ENOTSUP,
4818                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4819                                 "modifications of the VXLAN Network"
4820                                 " Identifier is not supported");
4821         if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI ||
4822             action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI)
4823                 return rte_flow_error_set(error, ENOTSUP,
4824                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4825                                 "modifications of the GENEVE Network"
4826                                 " Identifier is not supported");
4827         if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK ||
4828             action_modify_field->src.field == RTE_FLOW_FIELD_MARK ||
4829             action_modify_field->dst.field == RTE_FLOW_FIELD_META ||
4830             action_modify_field->src.field == RTE_FLOW_FIELD_META) {
4831                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4832                     !mlx5_flow_ext_mreg_supported(dev))
4833                         return rte_flow_error_set(error, ENOTSUP,
4834                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4835                                         "cannot modify mark or metadata without"
4836                                         " extended metadata register support");
4837         }
4838         if (action_modify_field->operation != RTE_FLOW_MODIFY_SET)
4839                 return rte_flow_error_set(error, ENOTSUP,
4840                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4841                                 "add and sub operations"
4842                                 " are not supported");
4843         return (action_modify_field->width / 32) +
4844                !!(action_modify_field->width % 32);
4845 }
4846
4847 /**
4848  * Validate jump action.
4849  *
4850  * @param[in] action
4851  *   Pointer to the jump action.
4852  * @param[in] action_flags
4853  *   Holds the actions detected until now.
4854  * @param[in] attributes
4855  *   Pointer to flow attributes
4856  * @param[in] external
4857  *   Action belongs to flow rule created by request external to PMD.
4858  * @param[out] error
4859  *   Pointer to error structure.
4860  *
4861  * @return
4862  *   0 on success, a negative errno value otherwise and rte_errno is set.
4863  */
4864 static int
4865 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
4866                              const struct mlx5_flow_tunnel *tunnel,
4867                              const struct rte_flow_action *action,
4868                              uint64_t action_flags,
4869                              const struct rte_flow_attr *attributes,
4870                              bool external, struct rte_flow_error *error)
4871 {
4872         uint32_t target_group, table;
4873         int ret = 0;
4874         struct flow_grp_info grp_info = {
4875                 .external = !!external,
4876                 .transfer = !!attributes->transfer,
4877                 .fdb_def_rule = 1,
4878                 .std_tbl_fix = 0
4879         };
4880         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4881                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4882                 return rte_flow_error_set(error, EINVAL,
4883                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4884                                           "can't have 2 fate actions in"
4885                                           " same flow");
4886         if (!action->conf)
4887                 return rte_flow_error_set(error, EINVAL,
4888                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4889                                           NULL, "action configuration not set");
4890         target_group =
4891                 ((const struct rte_flow_action_jump *)action->conf)->group;
4892         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
4893                                        &grp_info, error);
4894         if (ret)
4895                 return ret;
4896         if (attributes->group == target_group &&
4897             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
4898                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
4899                 return rte_flow_error_set(error, EINVAL,
4900                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4901                                           "target group must be other than"
4902                                           " the current flow group");
4903         return 0;
4904 }
4905
4906 /*
4907  * Validate the port_id action.
4908  *
4909  * @param[in] dev
4910  *   Pointer to rte_eth_dev structure.
4911  * @param[in] action_flags
4912  *   Bit-fields that holds the actions detected until now.
4913  * @param[in] action
4914  *   Port_id RTE action structure.
4915  * @param[in] attr
4916  *   Attributes of flow that includes this action.
4917  * @param[out] error
4918  *   Pointer to error structure.
4919  *
4920  * @return
4921  *   0 on success, a negative errno value otherwise and rte_errno is set.
4922  */
4923 static int
4924 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
4925                                 uint64_t action_flags,
4926                                 const struct rte_flow_action *action,
4927                                 const struct rte_flow_attr *attr,
4928                                 struct rte_flow_error *error)
4929 {
4930         const struct rte_flow_action_port_id *port_id;
4931         struct mlx5_priv *act_priv;
4932         struct mlx5_priv *dev_priv;
4933         uint16_t port;
4934
4935         if (!attr->transfer)
4936                 return rte_flow_error_set(error, ENOTSUP,
4937                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4938                                           NULL,
4939                                           "port id action is valid in transfer"
4940                                           " mode only");
4941         if (!action || !action->conf)
4942                 return rte_flow_error_set(error, ENOTSUP,
4943                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4944                                           NULL,
4945                                           "port id action parameters must be"
4946                                           " specified");
4947         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4948                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4949                 return rte_flow_error_set(error, EINVAL,
4950                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4951                                           "can have only one fate actions in"
4952                                           " a flow");
4953         dev_priv = mlx5_dev_to_eswitch_info(dev);
4954         if (!dev_priv)
4955                 return rte_flow_error_set(error, rte_errno,
4956                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4957                                           NULL,
4958                                           "failed to obtain E-Switch info");
4959         port_id = action->conf;
4960         port = port_id->original ? dev->data->port_id : port_id->id;
4961         act_priv = mlx5_port_to_eswitch_info(port, false);
4962         if (!act_priv)
4963                 return rte_flow_error_set
4964                                 (error, rte_errno,
4965                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
4966                                  "failed to obtain E-Switch port id for port");
4967         if (act_priv->domain_id != dev_priv->domain_id)
4968                 return rte_flow_error_set
4969                                 (error, EINVAL,
4970                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4971                                  "port does not belong to"
4972                                  " E-Switch being configured");
4973         return 0;
4974 }
4975
4976 /**
4977  * Get the maximum number of modify header actions.
4978  *
4979  * @param dev
4980  *   Pointer to rte_eth_dev structure.
4981  * @param flags
4982  *   Flags bits to check if root level.
4983  *
4984  * @return
4985  *   Max number of modify header actions device can support.
4986  */
4987 static inline unsigned int
4988 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
4989                               uint64_t flags)
4990 {
4991         /*
4992          * There's no way to directly query the max capacity from FW.
4993          * The maximal value on root table should be assumed to be supported.
4994          */
4995         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
4996                 return MLX5_MAX_MODIFY_NUM;
4997         else
4998                 return MLX5_ROOT_TBL_MODIFY_NUM;
4999 }
5000
5001 /**
5002  * Validate the meter action.
5003  *
5004  * @param[in] dev
5005  *   Pointer to rte_eth_dev structure.
5006  * @param[in] action_flags
5007  *   Bit-fields that holds the actions detected until now.
5008  * @param[in] action
5009  *   Pointer to the meter action.
5010  * @param[in] attr
5011  *   Attributes of flow that includes this action.
5012  * @param[out] error
5013  *   Pointer to error structure.
5014  *
5015  * @return
5016  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5017  */
5018 static int
5019 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
5020                                 uint64_t action_flags,
5021                                 const struct rte_flow_action *action,
5022                                 const struct rte_flow_attr *attr,
5023                                 bool *def_policy,
5024                                 struct rte_flow_error *error)
5025 {
5026         struct mlx5_priv *priv = dev->data->dev_private;
5027         const struct rte_flow_action_meter *am = action->conf;
5028         struct mlx5_flow_meter_info *fm;
5029         struct mlx5_flow_meter_policy *mtr_policy;
5030         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
5031
5032         if (!am)
5033                 return rte_flow_error_set(error, EINVAL,
5034                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5035                                           "meter action conf is NULL");
5036
5037         if (action_flags & MLX5_FLOW_ACTION_METER)
5038                 return rte_flow_error_set(error, ENOTSUP,
5039                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5040                                           "meter chaining not support");
5041         if (action_flags & MLX5_FLOW_ACTION_JUMP)
5042                 return rte_flow_error_set(error, ENOTSUP,
5043                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5044                                           "meter with jump not support");
5045         if (!priv->mtr_en)
5046                 return rte_flow_error_set(error, ENOTSUP,
5047                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5048                                           NULL,
5049                                           "meter action not supported");
5050         fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL);
5051         if (!fm)
5052                 return rte_flow_error_set(error, EINVAL,
5053                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5054                                           "Meter not found");
5055         /* aso meter can always be shared by different domains */
5056         if (fm->ref_cnt && !priv->sh->meter_aso_en &&
5057             !(fm->transfer == attr->transfer ||
5058               (!fm->ingress && !attr->ingress && attr->egress) ||
5059               (!fm->egress && !attr->egress && attr->ingress)))
5060                 return rte_flow_error_set(error, EINVAL,
5061                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5062                         "Flow attributes domain are either invalid "
5063                         "or have a domain conflict with current "
5064                         "meter attributes");
5065         if (fm->def_policy) {
5066                 if (!((attr->transfer &&
5067                         mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) ||
5068                         (attr->egress &&
5069                         mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) ||
5070                         (attr->ingress &&
5071                         mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS])))
5072                         return rte_flow_error_set(error, EINVAL,
5073                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5074                                           "Flow attributes domain "
5075                                           "have a conflict with current "
5076                                           "meter domain attributes");
5077                 *def_policy = true;
5078         } else {
5079                 mtr_policy = mlx5_flow_meter_policy_find(dev,
5080                                                 fm->policy_id, NULL);
5081                 if (!mtr_policy)
5082                         return rte_flow_error_set(error, EINVAL,
5083                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5084                                           "Invalid policy id for meter ");
5085                 if (!((attr->transfer && mtr_policy->transfer) ||
5086                         (attr->egress && mtr_policy->egress) ||
5087                         (attr->ingress && mtr_policy->ingress)))
5088                         return rte_flow_error_set(error, EINVAL,
5089                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5090                                           "Flow attributes domain "
5091                                           "have a conflict with current "
5092                                           "meter domain attributes");
5093                 *def_policy = false;
5094         }
5095         return 0;
5096 }
5097
5098 /**
5099  * Validate the age action.
5100  *
5101  * @param[in] action_flags
5102  *   Holds the actions detected until now.
5103  * @param[in] action
5104  *   Pointer to the age action.
5105  * @param[in] dev
5106  *   Pointer to the Ethernet device structure.
5107  * @param[out] error
5108  *   Pointer to error structure.
5109  *
5110  * @return
5111  *   0 on success, a negative errno value otherwise and rte_errno is set.
5112  */
5113 static int
5114 flow_dv_validate_action_age(uint64_t action_flags,
5115                             const struct rte_flow_action *action,
5116                             struct rte_eth_dev *dev,
5117                             struct rte_flow_error *error)
5118 {
5119         struct mlx5_priv *priv = dev->data->dev_private;
5120         const struct rte_flow_action_age *age = action->conf;
5121
5122         if (!priv->config.devx || (priv->sh->cmng.counter_fallback &&
5123             !priv->sh->aso_age_mng))
5124                 return rte_flow_error_set(error, ENOTSUP,
5125                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5126                                           NULL,
5127                                           "age action not supported");
5128         if (!(action->conf))
5129                 return rte_flow_error_set(error, EINVAL,
5130                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5131                                           "configuration cannot be null");
5132         if (!(age->timeout))
5133                 return rte_flow_error_set(error, EINVAL,
5134                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5135                                           "invalid timeout value 0");
5136         if (action_flags & MLX5_FLOW_ACTION_AGE)
5137                 return rte_flow_error_set(error, EINVAL,
5138                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5139                                           "duplicate age actions set");
5140         return 0;
5141 }
5142
5143 /**
5144  * Validate the modify-header IPv4 DSCP actions.
5145  *
5146  * @param[in] action_flags
5147  *   Holds the actions detected until now.
5148  * @param[in] action
5149  *   Pointer to the modify action.
5150  * @param[in] item_flags
5151  *   Holds the items detected.
5152  * @param[out] error
5153  *   Pointer to error structure.
5154  *
5155  * @return
5156  *   0 on success, a negative errno value otherwise and rte_errno is set.
5157  */
5158 static int
5159 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
5160                                          const struct rte_flow_action *action,
5161                                          const uint64_t item_flags,
5162                                          struct rte_flow_error *error)
5163 {
5164         int ret = 0;
5165
5166         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5167         if (!ret) {
5168                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
5169                         return rte_flow_error_set(error, EINVAL,
5170                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5171                                                   NULL,
5172                                                   "no ipv4 item in pattern");
5173         }
5174         return ret;
5175 }
5176
5177 /**
5178  * Validate the modify-header IPv6 DSCP actions.
5179  *
5180  * @param[in] action_flags
5181  *   Holds the actions detected until now.
5182  * @param[in] action
5183  *   Pointer to the modify action.
5184  * @param[in] item_flags
5185  *   Holds the items detected.
5186  * @param[out] error
5187  *   Pointer to error structure.
5188  *
5189  * @return
5190  *   0 on success, a negative errno value otherwise and rte_errno is set.
5191  */
5192 static int
5193 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
5194                                          const struct rte_flow_action *action,
5195                                          const uint64_t item_flags,
5196                                          struct rte_flow_error *error)
5197 {
5198         int ret = 0;
5199
5200         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5201         if (!ret) {
5202                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
5203                         return rte_flow_error_set(error, EINVAL,
5204                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5205                                                   NULL,
5206                                                   "no ipv6 item in pattern");
5207         }
5208         return ret;
5209 }
5210
5211 /**
5212  * Match modify-header resource.
5213  *
5214  * @param list
5215  *   Pointer to the hash list.
5216  * @param entry
5217  *   Pointer to exist resource entry object.
5218  * @param key
5219  *   Key of the new entry.
5220  * @param ctx
5221  *   Pointer to new modify-header resource.
5222  *
5223  * @return
5224  *   0 on matching, non-zero otherwise.
5225  */
5226 int
5227 flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
5228                         struct mlx5_hlist_entry *entry,
5229                         uint64_t key __rte_unused, void *cb_ctx)
5230 {
5231         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5232         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5233         struct mlx5_flow_dv_modify_hdr_resource *resource =
5234                         container_of(entry, typeof(*resource), entry);
5235         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5236
5237         key_len += ref->actions_num * sizeof(ref->actions[0]);
5238         return ref->actions_num != resource->actions_num ||
5239                memcmp(&ref->ft_type, &resource->ft_type, key_len);
5240 }
5241
5242 struct mlx5_hlist_entry *
5243 flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
5244                          void *cb_ctx)
5245 {
5246         struct mlx5_dev_ctx_shared *sh = list->ctx;
5247         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5248         struct mlx5dv_dr_domain *ns;
5249         struct mlx5_flow_dv_modify_hdr_resource *entry;
5250         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5251         int ret;
5252         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5253         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5254
5255         entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0,
5256                             SOCKET_ID_ANY);
5257         if (!entry) {
5258                 rte_flow_error_set(ctx->error, ENOMEM,
5259                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5260                                    "cannot allocate resource memory");
5261                 return NULL;
5262         }
5263         rte_memcpy(&entry->ft_type,
5264                    RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
5265                    key_len + data_len);
5266         if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
5267                 ns = sh->fdb_domain;
5268         else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
5269                 ns = sh->tx_domain;
5270         else
5271                 ns = sh->rx_domain;
5272         ret = mlx5_flow_os_create_flow_action_modify_header
5273                                         (sh->ctx, ns, entry,
5274                                          data_len, &entry->action);
5275         if (ret) {
5276                 mlx5_free(entry);
5277                 rte_flow_error_set(ctx->error, ENOMEM,
5278                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5279                                    NULL, "cannot create modification action");
5280                 return NULL;
5281         }
5282         return &entry->entry;
5283 }
5284
5285 /**
5286  * Validate the sample action.
5287  *
5288  * @param[in, out] action_flags
5289  *   Holds the actions detected until now.
5290  * @param[in] action
5291  *   Pointer to the sample action.
5292  * @param[in] dev
5293  *   Pointer to the Ethernet device structure.
5294  * @param[in] attr
5295  *   Attributes of flow that includes this action.
5296  * @param[in] item_flags
5297  *   Holds the items detected.
5298  * @param[in] rss
5299  *   Pointer to the RSS action.
5300  * @param[out] sample_rss
5301  *   Pointer to the RSS action in sample action list.
5302  * @param[out] count
5303  *   Pointer to the COUNT action in sample action list.
5304  * @param[out] fdb_mirror_limit
5305  *   Pointer to the FDB mirror limitation flag.
5306  * @param[out] error
5307  *   Pointer to error structure.
5308  *
5309  * @return
5310  *   0 on success, a negative errno value otherwise and rte_errno is set.
5311  */
5312 static int
5313 flow_dv_validate_action_sample(uint64_t *action_flags,
5314                                const struct rte_flow_action *action,
5315                                struct rte_eth_dev *dev,
5316                                const struct rte_flow_attr *attr,
5317                                uint64_t item_flags,
5318                                const struct rte_flow_action_rss *rss,
5319                                const struct rte_flow_action_rss **sample_rss,
5320                                const struct rte_flow_action_count **count,
5321                                int *fdb_mirror_limit,
5322                                struct rte_flow_error *error)
5323 {
5324         struct mlx5_priv *priv = dev->data->dev_private;
5325         struct mlx5_dev_config *dev_conf = &priv->config;
5326         const struct rte_flow_action_sample *sample = action->conf;
5327         const struct rte_flow_action *act;
5328         uint64_t sub_action_flags = 0;
5329         uint16_t queue_index = 0xFFFF;
5330         int actions_n = 0;
5331         int ret;
5332
5333         if (!sample)
5334                 return rte_flow_error_set(error, EINVAL,
5335                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5336                                           "configuration cannot be NULL");
5337         if (sample->ratio == 0)
5338                 return rte_flow_error_set(error, EINVAL,
5339                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5340                                           "ratio value starts from 1");
5341         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
5342                 return rte_flow_error_set(error, ENOTSUP,
5343                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5344                                           NULL,
5345                                           "sample action not supported");
5346         if (*action_flags & MLX5_FLOW_ACTION_SAMPLE)
5347                 return rte_flow_error_set(error, EINVAL,
5348                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5349                                           "Multiple sample actions not "
5350                                           "supported");
5351         if (*action_flags & MLX5_FLOW_ACTION_METER)
5352                 return rte_flow_error_set(error, EINVAL,
5353                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5354                                           "wrong action order, meter should "
5355                                           "be after sample action");
5356         if (*action_flags & MLX5_FLOW_ACTION_JUMP)
5357                 return rte_flow_error_set(error, EINVAL,
5358                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5359                                           "wrong action order, jump should "
5360                                           "be after sample action");
5361         act = sample->actions;
5362         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
5363                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5364                         return rte_flow_error_set(error, ENOTSUP,
5365                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5366                                                   act, "too many actions");
5367                 switch (act->type) {
5368                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5369                         ret = mlx5_flow_validate_action_queue(act,
5370                                                               sub_action_flags,
5371                                                               dev,
5372                                                               attr, error);
5373                         if (ret < 0)
5374                                 return ret;
5375                         queue_index = ((const struct rte_flow_action_queue *)
5376                                                         (act->conf))->index;
5377                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
5378                         ++actions_n;
5379                         break;
5380                 case RTE_FLOW_ACTION_TYPE_RSS:
5381                         *sample_rss = act->conf;
5382                         ret = mlx5_flow_validate_action_rss(act,
5383                                                             sub_action_flags,
5384                                                             dev, attr,
5385                                                             item_flags,
5386                                                             error);
5387                         if (ret < 0)
5388                                 return ret;
5389                         if (rss && *sample_rss &&
5390                             ((*sample_rss)->level != rss->level ||
5391                             (*sample_rss)->types != rss->types))
5392                                 return rte_flow_error_set(error, ENOTSUP,
5393                                         RTE_FLOW_ERROR_TYPE_ACTION,
5394                                         NULL,
5395                                         "Can't use the different RSS types "
5396                                         "or level in the same flow");
5397                         if (*sample_rss != NULL && (*sample_rss)->queue_num)
5398                                 queue_index = (*sample_rss)->queue[0];
5399                         sub_action_flags |= MLX5_FLOW_ACTION_RSS;
5400                         ++actions_n;
5401                         break;
5402                 case RTE_FLOW_ACTION_TYPE_MARK:
5403                         ret = flow_dv_validate_action_mark(dev, act,
5404                                                            sub_action_flags,
5405                                                            attr, error);
5406                         if (ret < 0)
5407                                 return ret;
5408                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
5409                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
5410                                                 MLX5_FLOW_ACTION_MARK_EXT;
5411                         else
5412                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
5413                         ++actions_n;
5414                         break;
5415                 case RTE_FLOW_ACTION_TYPE_COUNT:
5416                         ret = flow_dv_validate_action_count
5417                                 (dev, is_shared_action_count(act),
5418                                  *action_flags | sub_action_flags,
5419                                  error);
5420                         if (ret < 0)
5421                                 return ret;
5422                         *count = act->conf;
5423                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
5424                         *action_flags |= MLX5_FLOW_ACTION_COUNT;
5425                         ++actions_n;
5426                         break;
5427                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5428                         ret = flow_dv_validate_action_port_id(dev,
5429                                                               sub_action_flags,
5430                                                               act,
5431                                                               attr,
5432                                                               error);
5433                         if (ret)
5434                                 return ret;
5435                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5436                         ++actions_n;
5437                         break;
5438                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5439                         ret = flow_dv_validate_action_raw_encap_decap
5440                                 (dev, NULL, act->conf, attr, &sub_action_flags,
5441                                  &actions_n, action, item_flags, error);
5442                         if (ret < 0)
5443                                 return ret;
5444                         ++actions_n;
5445                         break;
5446                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5447                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5448                         ret = flow_dv_validate_action_l2_encap(dev,
5449                                                                sub_action_flags,
5450                                                                act, attr,
5451                                                                error);
5452                         if (ret < 0)
5453                                 return ret;
5454                         sub_action_flags |= MLX5_FLOW_ACTION_ENCAP;
5455                         ++actions_n;
5456                         break;
5457                 default:
5458                         return rte_flow_error_set(error, ENOTSUP,
5459                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5460                                                   NULL,
5461                                                   "Doesn't support optional "
5462                                                   "action");
5463                 }
5464         }
5465         if (attr->ingress && !attr->transfer) {
5466                 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE |
5467                                           MLX5_FLOW_ACTION_RSS)))
5468                         return rte_flow_error_set(error, EINVAL,
5469                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5470                                                   NULL,
5471                                                   "Ingress must has a dest "
5472                                                   "QUEUE for Sample");
5473         } else if (attr->egress && !attr->transfer) {
5474                 return rte_flow_error_set(error, ENOTSUP,
5475                                           RTE_FLOW_ERROR_TYPE_ACTION,
5476                                           NULL,
5477                                           "Sample Only support Ingress "
5478                                           "or E-Switch");
5479         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
5480                 MLX5_ASSERT(attr->transfer);
5481                 if (sample->ratio > 1)
5482                         return rte_flow_error_set(error, ENOTSUP,
5483                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5484                                                   NULL,
5485                                                   "E-Switch doesn't support "
5486                                                   "any optional action "
5487                                                   "for sampling");
5488                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
5489                         return rte_flow_error_set(error, ENOTSUP,
5490                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5491                                                   NULL,
5492                                                   "unsupported action QUEUE");
5493                 if (sub_action_flags & MLX5_FLOW_ACTION_RSS)
5494                         return rte_flow_error_set(error, ENOTSUP,
5495                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5496                                                   NULL,
5497                                                   "unsupported action QUEUE");
5498                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
5499                         return rte_flow_error_set(error, EINVAL,
5500                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5501                                                   NULL,
5502                                                   "E-Switch must has a dest "
5503                                                   "port for mirroring");
5504                 if (!priv->config.hca_attr.reg_c_preserve &&
5505                      priv->representor_id != -1)
5506                         *fdb_mirror_limit = 1;
5507         }
5508         /* Continue validation for Xcap actions.*/
5509         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
5510             (queue_index == 0xFFFF ||
5511              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5512                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5513                      MLX5_FLOW_XCAP_ACTIONS)
5514                         return rte_flow_error_set(error, ENOTSUP,
5515                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5516                                                   NULL, "encap and decap "
5517                                                   "combination aren't "
5518                                                   "supported");
5519                 if (!attr->transfer && attr->ingress && (sub_action_flags &
5520                                                         MLX5_FLOW_ACTION_ENCAP))
5521                         return rte_flow_error_set(error, ENOTSUP,
5522                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5523                                                   NULL, "encap is not supported"
5524                                                   " for ingress traffic");
5525         }
5526         return 0;
5527 }
5528
5529 /**
5530  * Find existing modify-header resource or create and register a new one.
5531  *
5532  * @param dev[in, out]
5533  *   Pointer to rte_eth_dev structure.
5534  * @param[in, out] resource
5535  *   Pointer to modify-header resource.
5536  * @parm[in, out] dev_flow
5537  *   Pointer to the dev_flow.
5538  * @param[out] error
5539  *   pointer to error structure.
5540  *
5541  * @return
5542  *   0 on success otherwise -errno and errno is set.
5543  */
5544 static int
5545 flow_dv_modify_hdr_resource_register
5546                         (struct rte_eth_dev *dev,
5547                          struct mlx5_flow_dv_modify_hdr_resource *resource,
5548                          struct mlx5_flow *dev_flow,
5549                          struct rte_flow_error *error)
5550 {
5551         struct mlx5_priv *priv = dev->data->dev_private;
5552         struct mlx5_dev_ctx_shared *sh = priv->sh;
5553         uint32_t key_len = sizeof(*resource) -
5554                            offsetof(typeof(*resource), ft_type) +
5555                            resource->actions_num * sizeof(resource->actions[0]);
5556         struct mlx5_hlist_entry *entry;
5557         struct mlx5_flow_cb_ctx ctx = {
5558                 .error = error,
5559                 .data = resource,
5560         };
5561         uint64_t key64;
5562
5563         resource->flags = dev_flow->dv.group ? 0 :
5564                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5565         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
5566                                     resource->flags))
5567                 return rte_flow_error_set(error, EOVERFLOW,
5568                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5569                                           "too many modify header items");
5570         key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
5571         entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
5572         if (!entry)
5573                 return -rte_errno;
5574         resource = container_of(entry, typeof(*resource), entry);
5575         dev_flow->handle->dvh.modify_hdr = resource;
5576         return 0;
5577 }
5578
5579 /**
5580  * Get DV flow counter by index.
5581  *
5582  * @param[in] dev
5583  *   Pointer to the Ethernet device structure.
5584  * @param[in] idx
5585  *   mlx5 flow counter index in the container.
5586  * @param[out] ppool
5587  *   mlx5 flow counter pool in the container.
5588  *
5589  * @return
5590  *   Pointer to the counter, NULL otherwise.
5591  */
5592 static struct mlx5_flow_counter *
5593 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
5594                            uint32_t idx,
5595                            struct mlx5_flow_counter_pool **ppool)
5596 {
5597         struct mlx5_priv *priv = dev->data->dev_private;
5598         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5599         struct mlx5_flow_counter_pool *pool;
5600
5601         /* Decrease to original index and clear shared bit. */
5602         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
5603         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
5604         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
5605         MLX5_ASSERT(pool);
5606         if (ppool)
5607                 *ppool = pool;
5608         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
5609 }
5610
5611 /**
5612  * Check the devx counter belongs to the pool.
5613  *
5614  * @param[in] pool
5615  *   Pointer to the counter pool.
5616  * @param[in] id
5617  *   The counter devx ID.
5618  *
5619  * @return
5620  *   True if counter belongs to the pool, false otherwise.
5621  */
5622 static bool
5623 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
5624 {
5625         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
5626                    MLX5_COUNTERS_PER_POOL;
5627
5628         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
5629                 return true;
5630         return false;
5631 }
5632
5633 /**
5634  * Get a pool by devx counter ID.
5635  *
5636  * @param[in] cmng
5637  *   Pointer to the counter management.
5638  * @param[in] id
5639  *   The counter devx ID.
5640  *
5641  * @return
5642  *   The counter pool pointer if exists, NULL otherwise,
5643  */
5644 static struct mlx5_flow_counter_pool *
5645 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
5646 {
5647         uint32_t i;
5648         struct mlx5_flow_counter_pool *pool = NULL;
5649
5650         rte_spinlock_lock(&cmng->pool_update_sl);
5651         /* Check last used pool. */
5652         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
5653             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
5654                 pool = cmng->pools[cmng->last_pool_idx];
5655                 goto out;
5656         }
5657         /* ID out of range means no suitable pool in the container. */
5658         if (id > cmng->max_id || id < cmng->min_id)
5659                 goto out;
5660         /*
5661          * Find the pool from the end of the container, since mostly counter
5662          * ID is sequence increasing, and the last pool should be the needed
5663          * one.
5664          */
5665         i = cmng->n_valid;
5666         while (i--) {
5667                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
5668
5669                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
5670                         pool = pool_tmp;
5671                         break;
5672                 }
5673         }
5674 out:
5675         rte_spinlock_unlock(&cmng->pool_update_sl);
5676         return pool;
5677 }
5678
5679 /**
5680  * Resize a counter container.
5681  *
5682  * @param[in] dev
5683  *   Pointer to the Ethernet device structure.
5684  *
5685  * @return
5686  *   0 on success, otherwise negative errno value and rte_errno is set.
5687  */
5688 static int
5689 flow_dv_container_resize(struct rte_eth_dev *dev)
5690 {
5691         struct mlx5_priv *priv = dev->data->dev_private;
5692         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5693         void *old_pools = cmng->pools;
5694         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
5695         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
5696         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
5697
5698         if (!pools) {
5699                 rte_errno = ENOMEM;
5700                 return -ENOMEM;
5701         }
5702         if (old_pools)
5703                 memcpy(pools, old_pools, cmng->n *
5704                                        sizeof(struct mlx5_flow_counter_pool *));
5705         cmng->n = resize;
5706         cmng->pools = pools;
5707         if (old_pools)
5708                 mlx5_free(old_pools);
5709         return 0;
5710 }
5711
5712 /**
5713  * Query a devx flow counter.
5714  *
5715  * @param[in] dev
5716  *   Pointer to the Ethernet device structure.
5717  * @param[in] counter
5718  *   Index to the flow counter.
5719  * @param[out] pkts
5720  *   The statistics value of packets.
5721  * @param[out] bytes
5722  *   The statistics value of bytes.
5723  *
5724  * @return
5725  *   0 on success, otherwise a negative errno value and rte_errno is set.
5726  */
5727 static inline int
5728 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
5729                      uint64_t *bytes)
5730 {
5731         struct mlx5_priv *priv = dev->data->dev_private;
5732         struct mlx5_flow_counter_pool *pool = NULL;
5733         struct mlx5_flow_counter *cnt;
5734         int offset;
5735
5736         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5737         MLX5_ASSERT(pool);
5738         if (priv->sh->cmng.counter_fallback)
5739                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
5740                                         0, pkts, bytes, 0, NULL, NULL, 0);
5741         rte_spinlock_lock(&pool->sl);
5742         if (!pool->raw) {
5743                 *pkts = 0;
5744                 *bytes = 0;
5745         } else {
5746                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
5747                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
5748                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
5749         }
5750         rte_spinlock_unlock(&pool->sl);
5751         return 0;
5752 }
5753
5754 /**
5755  * Create and initialize a new counter pool.
5756  *
5757  * @param[in] dev
5758  *   Pointer to the Ethernet device structure.
5759  * @param[out] dcs
5760  *   The devX counter handle.
5761  * @param[in] age
5762  *   Whether the pool is for counter that was allocated for aging.
5763  * @param[in/out] cont_cur
5764  *   Pointer to the container pointer, it will be update in pool resize.
5765  *
5766  * @return
5767  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
5768  */
5769 static struct mlx5_flow_counter_pool *
5770 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
5771                     uint32_t age)
5772 {
5773         struct mlx5_priv *priv = dev->data->dev_private;
5774         struct mlx5_flow_counter_pool *pool;
5775         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5776         bool fallback = priv->sh->cmng.counter_fallback;
5777         uint32_t size = sizeof(*pool);
5778
5779         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
5780         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
5781         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
5782         if (!pool) {
5783                 rte_errno = ENOMEM;
5784                 return NULL;
5785         }
5786         pool->raw = NULL;
5787         pool->is_aged = !!age;
5788         pool->query_gen = 0;
5789         pool->min_dcs = dcs;
5790         rte_spinlock_init(&pool->sl);
5791         rte_spinlock_init(&pool->csl);
5792         TAILQ_INIT(&pool->counters[0]);
5793         TAILQ_INIT(&pool->counters[1]);
5794         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
5795         rte_spinlock_lock(&cmng->pool_update_sl);
5796         pool->index = cmng->n_valid;
5797         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
5798                 mlx5_free(pool);
5799                 rte_spinlock_unlock(&cmng->pool_update_sl);
5800                 return NULL;
5801         }
5802         cmng->pools[pool->index] = pool;
5803         cmng->n_valid++;
5804         if (unlikely(fallback)) {
5805                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
5806
5807                 if (base < cmng->min_id)
5808                         cmng->min_id = base;
5809                 if (base > cmng->max_id)
5810                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
5811                 cmng->last_pool_idx = pool->index;
5812         }
5813         rte_spinlock_unlock(&cmng->pool_update_sl);
5814         return pool;
5815 }
5816
5817 /**
5818  * Prepare a new counter and/or a new counter pool.
5819  *
5820  * @param[in] dev
5821  *   Pointer to the Ethernet device structure.
5822  * @param[out] cnt_free
5823  *   Where to put the pointer of a new counter.
5824  * @param[in] age
5825  *   Whether the pool is for counter that was allocated for aging.
5826  *
5827  * @return
5828  *   The counter pool pointer and @p cnt_free is set on success,
5829  *   NULL otherwise and rte_errno is set.
5830  */
5831 static struct mlx5_flow_counter_pool *
5832 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
5833                              struct mlx5_flow_counter **cnt_free,
5834                              uint32_t age)
5835 {
5836         struct mlx5_priv *priv = dev->data->dev_private;
5837         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5838         struct mlx5_flow_counter_pool *pool;
5839         struct mlx5_counters tmp_tq;
5840         struct mlx5_devx_obj *dcs = NULL;
5841         struct mlx5_flow_counter *cnt;
5842         enum mlx5_counter_type cnt_type =
5843                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
5844         bool fallback = priv->sh->cmng.counter_fallback;
5845         uint32_t i;
5846
5847         if (fallback) {
5848                 /* bulk_bitmap must be 0 for single counter allocation. */
5849                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
5850                 if (!dcs)
5851                         return NULL;
5852                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
5853                 if (!pool) {
5854                         pool = flow_dv_pool_create(dev, dcs, age);
5855                         if (!pool) {
5856                                 mlx5_devx_cmd_destroy(dcs);
5857                                 return NULL;
5858                         }
5859                 }
5860                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
5861                 cnt = MLX5_POOL_GET_CNT(pool, i);
5862                 cnt->pool = pool;
5863                 cnt->dcs_when_free = dcs;
5864                 *cnt_free = cnt;
5865                 return pool;
5866         }
5867         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
5868         if (!dcs) {
5869                 rte_errno = ENODATA;
5870                 return NULL;
5871         }
5872         pool = flow_dv_pool_create(dev, dcs, age);
5873         if (!pool) {
5874                 mlx5_devx_cmd_destroy(dcs);
5875                 return NULL;
5876         }
5877         TAILQ_INIT(&tmp_tq);
5878         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
5879                 cnt = MLX5_POOL_GET_CNT(pool, i);
5880                 cnt->pool = pool;
5881                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
5882         }
5883         rte_spinlock_lock(&cmng->csl[cnt_type]);
5884         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
5885         rte_spinlock_unlock(&cmng->csl[cnt_type]);
5886         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
5887         (*cnt_free)->pool = pool;
5888         return pool;
5889 }
5890
5891 /**
5892  * Allocate a flow counter.
5893  *
5894  * @param[in] dev
5895  *   Pointer to the Ethernet device structure.
5896  * @param[in] age
5897  *   Whether the counter was allocated for aging.
5898  *
5899  * @return
5900  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5901  */
5902 static uint32_t
5903 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
5904 {
5905         struct mlx5_priv *priv = dev->data->dev_private;
5906         struct mlx5_flow_counter_pool *pool = NULL;
5907         struct mlx5_flow_counter *cnt_free = NULL;
5908         bool fallback = priv->sh->cmng.counter_fallback;
5909         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5910         enum mlx5_counter_type cnt_type =
5911                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
5912         uint32_t cnt_idx;
5913
5914         if (!priv->config.devx) {
5915                 rte_errno = ENOTSUP;
5916                 return 0;
5917         }
5918         /* Get free counters from container. */
5919         rte_spinlock_lock(&cmng->csl[cnt_type]);
5920         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
5921         if (cnt_free)
5922                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
5923         rte_spinlock_unlock(&cmng->csl[cnt_type]);
5924         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
5925                 goto err;
5926         pool = cnt_free->pool;
5927         if (fallback)
5928                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
5929         /* Create a DV counter action only in the first time usage. */
5930         if (!cnt_free->action) {
5931                 uint16_t offset;
5932                 struct mlx5_devx_obj *dcs;
5933                 int ret;
5934
5935                 if (!fallback) {
5936                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
5937                         dcs = pool->min_dcs;
5938                 } else {
5939                         offset = 0;
5940                         dcs = cnt_free->dcs_when_free;
5941                 }
5942                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
5943                                                             &cnt_free->action);
5944                 if (ret) {
5945                         rte_errno = errno;
5946                         goto err;
5947                 }
5948         }
5949         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
5950                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
5951         /* Update the counter reset values. */
5952         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
5953                                  &cnt_free->bytes))
5954                 goto err;
5955         if (!fallback && !priv->sh->cmng.query_thread_on)
5956                 /* Start the asynchronous batch query by the host thread. */
5957                 mlx5_set_query_alarm(priv->sh);
5958         /*
5959          * When the count action isn't shared (by ID), shared_info field is
5960          * used for indirect action API's refcnt.
5961          * When the counter action is not shared neither by ID nor by indirect
5962          * action API, shared info must be 1.
5963          */
5964         cnt_free->shared_info.refcnt = 1;
5965         return cnt_idx;
5966 err:
5967         if (cnt_free) {
5968                 cnt_free->pool = pool;
5969                 if (fallback)
5970                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
5971                 rte_spinlock_lock(&cmng->csl[cnt_type]);
5972                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
5973                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
5974         }
5975         return 0;
5976 }
5977
5978 /**
5979  * Allocate a shared flow counter.
5980  *
5981  * @param[in] ctx
5982  *   Pointer to the shared counter configuration.
5983  * @param[in] data
5984  *   Pointer to save the allocated counter index.
5985  *
5986  * @return
5987  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5988  */
5989
5990 static int32_t
5991 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
5992 {
5993         struct mlx5_shared_counter_conf *conf = ctx;
5994         struct rte_eth_dev *dev = conf->dev;
5995         struct mlx5_flow_counter *cnt;
5996
5997         data->dword = flow_dv_counter_alloc(dev, 0);
5998         data->dword |= MLX5_CNT_SHARED_OFFSET;
5999         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
6000         cnt->shared_info.id = conf->id;
6001         return 0;
6002 }
6003
6004 /**
6005  * Get a shared flow counter.
6006  *
6007  * @param[in] dev
6008  *   Pointer to the Ethernet device structure.
6009  * @param[in] id
6010  *   Counter identifier.
6011  *
6012  * @return
6013  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6014  */
6015 static uint32_t
6016 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
6017 {
6018         struct mlx5_priv *priv = dev->data->dev_private;
6019         struct mlx5_shared_counter_conf conf = {
6020                 .dev = dev,
6021                 .id = id,
6022         };
6023         union mlx5_l3t_data data = {
6024                 .dword = 0,
6025         };
6026
6027         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
6028                                flow_dv_counter_alloc_shared_cb, &conf);
6029         return data.dword;
6030 }
6031
6032 /**
6033  * Get age param from counter index.
6034  *
6035  * @param[in] dev
6036  *   Pointer to the Ethernet device structure.
6037  * @param[in] counter
6038  *   Index to the counter handler.
6039  *
6040  * @return
6041  *   The aging parameter specified for the counter index.
6042  */
6043 static struct mlx5_age_param*
6044 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
6045                                 uint32_t counter)
6046 {
6047         struct mlx5_flow_counter *cnt;
6048         struct mlx5_flow_counter_pool *pool = NULL;
6049
6050         flow_dv_counter_get_by_idx(dev, counter, &pool);
6051         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
6052         cnt = MLX5_POOL_GET_CNT(pool, counter);
6053         return MLX5_CNT_TO_AGE(cnt);
6054 }
6055
6056 /**
6057  * Remove a flow counter from aged counter list.
6058  *
6059  * @param[in] dev
6060  *   Pointer to the Ethernet device structure.
6061  * @param[in] counter
6062  *   Index to the counter handler.
6063  * @param[in] cnt
6064  *   Pointer to the counter handler.
6065  */
6066 static void
6067 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
6068                                 uint32_t counter, struct mlx5_flow_counter *cnt)
6069 {
6070         struct mlx5_age_info *age_info;
6071         struct mlx5_age_param *age_param;
6072         struct mlx5_priv *priv = dev->data->dev_private;
6073         uint16_t expected = AGE_CANDIDATE;
6074
6075         age_info = GET_PORT_AGE_INFO(priv);
6076         age_param = flow_dv_counter_idx_get_age(dev, counter);
6077         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
6078                                          AGE_FREE, false, __ATOMIC_RELAXED,
6079                                          __ATOMIC_RELAXED)) {
6080                 /**
6081                  * We need the lock even it is age timeout,
6082                  * since counter may still in process.
6083                  */
6084                 rte_spinlock_lock(&age_info->aged_sl);
6085                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
6086                 rte_spinlock_unlock(&age_info->aged_sl);
6087                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
6088         }
6089 }
6090
6091 /**
6092  * Release a flow counter.
6093  *
6094  * @param[in] dev
6095  *   Pointer to the Ethernet device structure.
6096  * @param[in] counter
6097  *   Index to the counter handler.
6098  */
6099 static void
6100 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter)
6101 {
6102         struct mlx5_priv *priv = dev->data->dev_private;
6103         struct mlx5_flow_counter_pool *pool = NULL;
6104         struct mlx5_flow_counter *cnt;
6105         enum mlx5_counter_type cnt_type;
6106
6107         if (!counter)
6108                 return;
6109         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6110         MLX5_ASSERT(pool);
6111         /*
6112          * If the counter action is shared by ID, the l3t_clear_entry function
6113          * reduces its references counter. If after the reduction the action is
6114          * still referenced, the function returns here and does not release it.
6115          */
6116         if (IS_LEGACY_SHARED_CNT(counter) &&
6117             mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
6118                 return;
6119         /*
6120          * If the counter action is shared by indirect action API, the atomic
6121          * function reduces its references counter. If after the reduction the
6122          * action is still referenced, the function returns here and does not
6123          * release it.
6124          * When the counter action is not shared neither by ID nor by indirect
6125          * action API, shared info is 1 before the reduction, so this condition
6126          * is failed and function doesn't return here.
6127          */
6128         if (!IS_LEGACY_SHARED_CNT(counter) &&
6129             __atomic_sub_fetch(&cnt->shared_info.refcnt, 1, __ATOMIC_RELAXED))
6130                 return;
6131         if (pool->is_aged)
6132                 flow_dv_counter_remove_from_age(dev, counter, cnt);
6133         cnt->pool = pool;
6134         /*
6135          * Put the counter back to list to be updated in none fallback mode.
6136          * Currently, we are using two list alternately, while one is in query,
6137          * add the freed counter to the other list based on the pool query_gen
6138          * value. After query finishes, add counter the list to the global
6139          * container counter list. The list changes while query starts. In
6140          * this case, lock will not be needed as query callback and release
6141          * function both operate with the different list.
6142          */
6143         if (!priv->sh->cmng.counter_fallback) {
6144                 rte_spinlock_lock(&pool->csl);
6145                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
6146                 rte_spinlock_unlock(&pool->csl);
6147         } else {
6148                 cnt->dcs_when_free = cnt->dcs_when_active;
6149                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6150                                            MLX5_COUNTER_TYPE_ORIGIN;
6151                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
6152                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
6153                                   cnt, next);
6154                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
6155         }
6156 }
6157
6158 /**
6159  * Resize a meter id container.
6160  *
6161  * @param[in] dev
6162  *   Pointer to the Ethernet device structure.
6163  *
6164  * @return
6165  *   0 on success, otherwise negative errno value and rte_errno is set.
6166  */
6167 static int
6168 flow_dv_mtr_container_resize(struct rte_eth_dev *dev)
6169 {
6170         struct mlx5_priv *priv = dev->data->dev_private;
6171         struct mlx5_aso_mtr_pools_mng *pools_mng =
6172                                 &priv->sh->mtrmng->pools_mng;
6173         void *old_pools = pools_mng->pools;
6174         uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE;
6175         uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize;
6176         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
6177
6178         if (!pools) {
6179                 rte_errno = ENOMEM;
6180                 return -ENOMEM;
6181         }
6182         if (!pools_mng->n)
6183                 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) {
6184                         mlx5_free(pools);
6185                         return -ENOMEM;
6186                 }
6187         if (old_pools)
6188                 memcpy(pools, old_pools, pools_mng->n *
6189                                        sizeof(struct mlx5_aso_mtr_pool *));
6190         pools_mng->n = resize;
6191         pools_mng->pools = pools;
6192         if (old_pools)
6193                 mlx5_free(old_pools);
6194         return 0;
6195 }
6196
6197 /**
6198  * Prepare a new meter and/or a new meter pool.
6199  *
6200  * @param[in] dev
6201  *   Pointer to the Ethernet device structure.
6202  * @param[out] mtr_free
6203  *   Where to put the pointer of a new meter.g.
6204  *
6205  * @return
6206  *   The meter pool pointer and @mtr_free is set on success,
6207  *   NULL otherwise and rte_errno is set.
6208  */
6209 static struct mlx5_aso_mtr_pool *
6210 flow_dv_mtr_pool_create(struct rte_eth_dev *dev,
6211                              struct mlx5_aso_mtr **mtr_free)
6212 {
6213         struct mlx5_priv *priv = dev->data->dev_private;
6214         struct mlx5_aso_mtr_pools_mng *pools_mng =
6215                                 &priv->sh->mtrmng->pools_mng;
6216         struct mlx5_aso_mtr_pool *pool = NULL;
6217         struct mlx5_devx_obj *dcs = NULL;
6218         uint32_t i;
6219         uint32_t log_obj_size;
6220
6221         log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
6222         dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx,
6223                         priv->sh->pdn, log_obj_size);
6224         if (!dcs) {
6225                 rte_errno = ENODATA;
6226                 return NULL;
6227         }
6228         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
6229         if (!pool) {
6230                 rte_errno = ENOMEM;
6231                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6232                 return NULL;
6233         }
6234         pool->devx_obj = dcs;
6235         pool->index = pools_mng->n_valid;
6236         if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) {
6237                 mlx5_free(pool);
6238                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6239                 return NULL;
6240         }
6241         pools_mng->pools[pool->index] = pool;
6242         pools_mng->n_valid++;
6243         for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) {
6244                 pool->mtrs[i].offset = i;
6245                 LIST_INSERT_HEAD(&pools_mng->meters,
6246                                                 &pool->mtrs[i], next);
6247         }
6248         pool->mtrs[0].offset = 0;
6249         *mtr_free = &pool->mtrs[0];
6250         return pool;
6251 }
6252
6253 /**
6254  * Release a flow meter into pool.
6255  *
6256  * @param[in] dev
6257  *   Pointer to the Ethernet device structure.
6258  * @param[in] mtr_idx
6259  *   Index to aso flow meter.
6260  */
6261 static void
6262 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx)
6263 {
6264         struct mlx5_priv *priv = dev->data->dev_private;
6265         struct mlx5_aso_mtr_pools_mng *pools_mng =
6266                                 &priv->sh->mtrmng->pools_mng;
6267         struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
6268
6269         MLX5_ASSERT(aso_mtr);
6270         rte_spinlock_lock(&pools_mng->mtrsl);
6271         memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info));
6272         aso_mtr->state = ASO_METER_FREE;
6273         LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next);
6274         rte_spinlock_unlock(&pools_mng->mtrsl);
6275 }
6276
6277 /**
6278  * Allocate a aso flow meter.
6279  *
6280  * @param[in] dev
6281  *   Pointer to the Ethernet device structure.
6282  *
6283  * @return
6284  *   Index to aso flow meter on success, 0 otherwise and rte_errno is set.
6285  */
6286 static uint32_t
6287 flow_dv_mtr_alloc(struct rte_eth_dev *dev)
6288 {
6289         struct mlx5_priv *priv = dev->data->dev_private;
6290         struct mlx5_aso_mtr *mtr_free = NULL;
6291         struct mlx5_aso_mtr_pools_mng *pools_mng =
6292                                 &priv->sh->mtrmng->pools_mng;
6293         struct mlx5_aso_mtr_pool *pool;
6294         uint32_t mtr_idx = 0;
6295
6296         if (!priv->config.devx) {
6297                 rte_errno = ENOTSUP;
6298                 return 0;
6299         }
6300         /* Allocate the flow meter memory. */
6301         /* Get free meters from management. */
6302         rte_spinlock_lock(&pools_mng->mtrsl);
6303         mtr_free = LIST_FIRST(&pools_mng->meters);
6304         if (mtr_free)
6305                 LIST_REMOVE(mtr_free, next);
6306         if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) {
6307                 rte_spinlock_unlock(&pools_mng->mtrsl);
6308                 return 0;
6309         }
6310         mtr_free->state = ASO_METER_WAIT;
6311         rte_spinlock_unlock(&pools_mng->mtrsl);
6312         pool = container_of(mtr_free,
6313                         struct mlx5_aso_mtr_pool,
6314                         mtrs[mtr_free->offset]);
6315         mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
6316         if (!mtr_free->fm.meter_action) {
6317 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
6318                 struct rte_flow_error error;
6319                 uint8_t reg_id;
6320
6321                 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
6322                 mtr_free->fm.meter_action =
6323                         mlx5_glue->dv_create_flow_action_aso
6324                                                 (priv->sh->rx_domain,
6325                                                  pool->devx_obj->obj,
6326                                                  mtr_free->offset,
6327                                                  (1 << MLX5_FLOW_COLOR_GREEN),
6328                                                  reg_id - REG_C_0);
6329 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
6330                 if (!mtr_free->fm.meter_action) {
6331                         flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
6332                         return 0;
6333                 }
6334         }
6335         return mtr_idx;
6336 }
6337
6338 /**
6339  * Verify the @p attributes will be correctly understood by the NIC and store
6340  * them in the @p flow if everything is correct.
6341  *
6342  * @param[in] dev
6343  *   Pointer to dev struct.
6344  * @param[in] attributes
6345  *   Pointer to flow attributes
6346  * @param[in] external
6347  *   This flow rule is created by request external to PMD.
6348  * @param[out] error
6349  *   Pointer to error structure.
6350  *
6351  * @return
6352  *   - 0 on success and non root table.
6353  *   - 1 on success and root table.
6354  *   - a negative errno value otherwise and rte_errno is set.
6355  */
6356 static int
6357 flow_dv_validate_attributes(struct rte_eth_dev *dev,
6358                             const struct mlx5_flow_tunnel *tunnel,
6359                             const struct rte_flow_attr *attributes,
6360                             const struct flow_grp_info *grp_info,
6361                             struct rte_flow_error *error)
6362 {
6363         struct mlx5_priv *priv = dev->data->dev_private;
6364         uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes);
6365         int ret = 0;
6366
6367 #ifndef HAVE_MLX5DV_DR
6368         RTE_SET_USED(tunnel);
6369         RTE_SET_USED(grp_info);
6370         if (attributes->group)
6371                 return rte_flow_error_set(error, ENOTSUP,
6372                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6373                                           NULL,
6374                                           "groups are not supported");
6375 #else
6376         uint32_t table = 0;
6377
6378         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
6379                                        grp_info, error);
6380         if (ret)
6381                 return ret;
6382         if (!table)
6383                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
6384 #endif
6385         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
6386             attributes->priority > lowest_priority)
6387                 return rte_flow_error_set(error, ENOTSUP,
6388                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
6389                                           NULL,
6390                                           "priority out of range");
6391         if (attributes->transfer) {
6392                 if (!priv->config.dv_esw_en)
6393                         return rte_flow_error_set
6394                                 (error, ENOTSUP,
6395                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6396                                  "E-Switch dr is not supported");
6397                 if (!(priv->representor || priv->master))
6398                         return rte_flow_error_set
6399                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6400                                  NULL, "E-Switch configuration can only be"
6401                                  " done by a master or a representor device");
6402                 if (attributes->egress)
6403                         return rte_flow_error_set
6404                                 (error, ENOTSUP,
6405                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
6406                                  "egress is not supported");
6407         }
6408         if (!(attributes->egress ^ attributes->ingress))
6409                 return rte_flow_error_set(error, ENOTSUP,
6410                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
6411                                           "must specify exactly one of "
6412                                           "ingress or egress");
6413         return ret;
6414 }
6415
6416 static uint16_t
6417 mlx5_flow_locate_proto_l3(const struct rte_flow_item **head,
6418                           const struct rte_flow_item *end)
6419 {
6420         const struct rte_flow_item *item = *head;
6421         uint16_t l3_protocol;
6422
6423         for (; item != end; item++) {
6424                 switch (item->type) {
6425                 default:
6426                         break;
6427                 case RTE_FLOW_ITEM_TYPE_IPV4:
6428                         l3_protocol = RTE_ETHER_TYPE_IPV4;
6429                         goto l3_ok;
6430                 case RTE_FLOW_ITEM_TYPE_IPV6:
6431                         l3_protocol = RTE_ETHER_TYPE_IPV6;
6432                         goto l3_ok;
6433                 case RTE_FLOW_ITEM_TYPE_ETH:
6434                         if (item->mask && item->spec) {
6435                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_eth,
6436                                                             type, item,
6437                                                             l3_protocol);
6438                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6439                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6440                                         goto l3_ok;
6441                         }
6442                         break;
6443                 case RTE_FLOW_ITEM_TYPE_VLAN:
6444                         if (item->mask && item->spec) {
6445                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_vlan,
6446                                                             inner_type, item,
6447                                                             l3_protocol);
6448                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6449                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6450                                         goto l3_ok;
6451                         }
6452                         break;
6453                 }
6454         }
6455         return 0;
6456 l3_ok:
6457         *head = item;
6458         return l3_protocol;
6459 }
6460
6461 static uint8_t
6462 mlx5_flow_locate_proto_l4(const struct rte_flow_item **head,
6463                           const struct rte_flow_item *end)
6464 {
6465         const struct rte_flow_item *item = *head;
6466         uint8_t l4_protocol;
6467
6468         for (; item != end; item++) {
6469                 switch (item->type) {
6470                 default:
6471                         break;
6472                 case RTE_FLOW_ITEM_TYPE_TCP:
6473                         l4_protocol = IPPROTO_TCP;
6474                         goto l4_ok;
6475                 case RTE_FLOW_ITEM_TYPE_UDP:
6476                         l4_protocol = IPPROTO_UDP;
6477                         goto l4_ok;
6478                 case RTE_FLOW_ITEM_TYPE_IPV4:
6479                         if (item->mask && item->spec) {
6480                                 const struct rte_flow_item_ipv4 *mask, *spec;
6481
6482                                 mask = (typeof(mask))item->mask;
6483                                 spec = (typeof(spec))item->spec;
6484                                 l4_protocol = mask->hdr.next_proto_id &
6485                                               spec->hdr.next_proto_id;
6486                                 if (l4_protocol == IPPROTO_TCP ||
6487                                     l4_protocol == IPPROTO_UDP)
6488                                         goto l4_ok;
6489                         }
6490                         break;
6491                 case RTE_FLOW_ITEM_TYPE_IPV6:
6492                         if (item->mask && item->spec) {
6493                                 const struct rte_flow_item_ipv6 *mask, *spec;
6494                                 mask = (typeof(mask))item->mask;
6495                                 spec = (typeof(spec))item->spec;
6496                                 l4_protocol = mask->hdr.proto & spec->hdr.proto;
6497                                 if (l4_protocol == IPPROTO_TCP ||
6498                                     l4_protocol == IPPROTO_UDP)
6499                                         goto l4_ok;
6500                         }
6501                         break;
6502                 }
6503         }
6504         return 0;
6505 l4_ok:
6506         *head = item;
6507         return l4_protocol;
6508 }
6509
6510 static int
6511 flow_dv_validate_item_integrity(struct rte_eth_dev *dev,
6512                                 const struct rte_flow_item *rule_items,
6513                                 const struct rte_flow_item *integrity_item,
6514                                 struct rte_flow_error *error)
6515 {
6516         struct mlx5_priv *priv = dev->data->dev_private;
6517         const struct rte_flow_item *tunnel_item, *end_item, *item = rule_items;
6518         const struct rte_flow_item_integrity *mask = (typeof(mask))
6519                                                      integrity_item->mask;
6520         const struct rte_flow_item_integrity *spec = (typeof(spec))
6521                                                      integrity_item->spec;
6522         uint32_t protocol;
6523
6524         if (!priv->config.hca_attr.pkt_integrity_match)
6525                 return rte_flow_error_set(error, ENOTSUP,
6526                                           RTE_FLOW_ERROR_TYPE_ITEM,
6527                                           integrity_item,
6528                                           "packet integrity integrity_item not supported");
6529         if (!mask)
6530                 mask = &rte_flow_item_integrity_mask;
6531         if (!mlx5_validate_integrity_item(mask))
6532                 return rte_flow_error_set(error, ENOTSUP,
6533                                           RTE_FLOW_ERROR_TYPE_ITEM,
6534                                           integrity_item,
6535                                           "unsupported integrity filter");
6536         tunnel_item = mlx5_flow_find_tunnel_item(rule_items);
6537         if (spec->level > 1) {
6538                 if (!tunnel_item)
6539                         return rte_flow_error_set(error, ENOTSUP,
6540                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6541                                                   integrity_item,
6542                                                   "missing tunnel item");
6543                 item = tunnel_item;
6544                 end_item = mlx5_find_end_item(tunnel_item);
6545         } else {
6546                 end_item = tunnel_item ? tunnel_item :
6547                            mlx5_find_end_item(integrity_item);
6548         }
6549         if (mask->l3_ok || mask->ipv4_csum_ok) {
6550                 protocol = mlx5_flow_locate_proto_l3(&item, end_item);
6551                 if (!protocol)
6552                         return rte_flow_error_set(error, EINVAL,
6553                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6554                                                   integrity_item,
6555                                                   "missing L3 protocol");
6556         }
6557         if (mask->l4_ok || mask->l4_csum_ok) {
6558                 protocol = mlx5_flow_locate_proto_l4(&item, end_item);
6559                 if (!protocol)
6560                         return rte_flow_error_set(error, EINVAL,
6561                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6562                                                   integrity_item,
6563                                                   "missing L4 protocol");
6564         }
6565         return 0;
6566 }
6567
6568 /**
6569  * Internal validation function. For validating both actions and items.
6570  *
6571  * @param[in] dev
6572  *   Pointer to the rte_eth_dev structure.
6573  * @param[in] attr
6574  *   Pointer to the flow attributes.
6575  * @param[in] items
6576  *   Pointer to the list of items.
6577  * @param[in] actions
6578  *   Pointer to the list of actions.
6579  * @param[in] external
6580  *   This flow rule is created by request external to PMD.
6581  * @param[in] hairpin
6582  *   Number of hairpin TX actions, 0 means classic flow.
6583  * @param[out] error
6584  *   Pointer to the error structure.
6585  *
6586  * @return
6587  *   0 on success, a negative errno value otherwise and rte_errno is set.
6588  */
6589 static int
6590 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
6591                  const struct rte_flow_item items[],
6592                  const struct rte_flow_action actions[],
6593                  bool external, int hairpin, struct rte_flow_error *error)
6594 {
6595         int ret;
6596         uint64_t action_flags = 0;
6597         uint64_t item_flags = 0;
6598         uint64_t last_item = 0;
6599         uint8_t next_protocol = 0xff;
6600         uint16_t ether_type = 0;
6601         int actions_n = 0;
6602         uint8_t item_ipv6_proto = 0;
6603         int fdb_mirror_limit = 0;
6604         int modify_after_mirror = 0;
6605         const struct rte_flow_item *geneve_item = NULL;
6606         const struct rte_flow_item *gre_item = NULL;
6607         const struct rte_flow_item *gtp_item = NULL;
6608         const struct rte_flow_action_raw_decap *decap;
6609         const struct rte_flow_action_raw_encap *encap;
6610         const struct rte_flow_action_rss *rss = NULL;
6611         const struct rte_flow_action_rss *sample_rss = NULL;
6612         const struct rte_flow_action_count *sample_count = NULL;
6613         const struct rte_flow_item_tcp nic_tcp_mask = {
6614                 .hdr = {
6615                         .tcp_flags = 0xFF,
6616                         .src_port = RTE_BE16(UINT16_MAX),
6617                         .dst_port = RTE_BE16(UINT16_MAX),
6618                 }
6619         };
6620         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
6621                 .hdr = {
6622                         .src_addr =
6623                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6624                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6625                         .dst_addr =
6626                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6627                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6628                         .vtc_flow = RTE_BE32(0xffffffff),
6629                         .proto = 0xff,
6630                         .hop_limits = 0xff,
6631                 },
6632                 .has_frag_ext = 1,
6633         };
6634         const struct rte_flow_item_ecpri nic_ecpri_mask = {
6635                 .hdr = {
6636                         .common = {
6637                                 .u32 =
6638                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
6639                                         .type = 0xFF,
6640                                         }).u32),
6641                         },
6642                         .dummy[0] = 0xffffffff,
6643                 },
6644         };
6645         struct mlx5_priv *priv = dev->data->dev_private;
6646         struct mlx5_dev_config *dev_conf = &priv->config;
6647         uint16_t queue_index = 0xFFFF;
6648         const struct rte_flow_item_vlan *vlan_m = NULL;
6649         uint32_t rw_act_num = 0;
6650         uint64_t is_root;
6651         const struct mlx5_flow_tunnel *tunnel;
6652         enum mlx5_tof_rule_type tof_rule_type;
6653         struct flow_grp_info grp_info = {
6654                 .external = !!external,
6655                 .transfer = !!attr->transfer,
6656                 .fdb_def_rule = !!priv->fdb_def_rule,
6657                 .std_tbl_fix = true,
6658         };
6659         const struct rte_eth_hairpin_conf *conf;
6660         const struct rte_flow_item *rule_items = items;
6661         bool def_policy = false;
6662
6663         if (items == NULL)
6664                 return -1;
6665         tunnel = is_tunnel_offload_active(dev) ?
6666                  mlx5_get_tof(items, actions, &tof_rule_type) : NULL;
6667         if (tunnel) {
6668                 if (priv->representor)
6669                         return rte_flow_error_set
6670                                 (error, ENOTSUP,
6671                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6672                                  NULL, "decap not supported for VF representor");
6673                 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE)
6674                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6675                 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE)
6676                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
6677                                         MLX5_FLOW_ACTION_DECAP;
6678                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
6679                                         (dev, attr, tunnel, tof_rule_type);
6680         }
6681         ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error);
6682         if (ret < 0)
6683                 return ret;
6684         is_root = (uint64_t)ret;
6685         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6686                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6687                 int type = items->type;
6688
6689                 if (!mlx5_flow_os_item_supported(type))
6690                         return rte_flow_error_set(error, ENOTSUP,
6691                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6692                                                   NULL, "item not supported");
6693                 switch (type) {
6694                 case RTE_FLOW_ITEM_TYPE_VOID:
6695                         break;
6696                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6697                         ret = flow_dv_validate_item_port_id
6698                                         (dev, items, attr, item_flags, error);
6699                         if (ret < 0)
6700                                 return ret;
6701                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6702                         break;
6703                 case RTE_FLOW_ITEM_TYPE_ETH:
6704                         ret = mlx5_flow_validate_item_eth(items, item_flags,
6705                                                           true, error);
6706                         if (ret < 0)
6707                                 return ret;
6708                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
6709                                              MLX5_FLOW_LAYER_OUTER_L2;
6710                         if (items->mask != NULL && items->spec != NULL) {
6711                                 ether_type =
6712                                         ((const struct rte_flow_item_eth *)
6713                                          items->spec)->type;
6714                                 ether_type &=
6715                                         ((const struct rte_flow_item_eth *)
6716                                          items->mask)->type;
6717                                 ether_type = rte_be_to_cpu_16(ether_type);
6718                         } else {
6719                                 ether_type = 0;
6720                         }
6721                         break;
6722                 case RTE_FLOW_ITEM_TYPE_VLAN:
6723                         ret = flow_dv_validate_item_vlan(items, item_flags,
6724                                                          dev, error);
6725                         if (ret < 0)
6726                                 return ret;
6727                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
6728                                              MLX5_FLOW_LAYER_OUTER_VLAN;
6729                         if (items->mask != NULL && items->spec != NULL) {
6730                                 ether_type =
6731                                         ((const struct rte_flow_item_vlan *)
6732                                          items->spec)->inner_type;
6733                                 ether_type &=
6734                                         ((const struct rte_flow_item_vlan *)
6735                                          items->mask)->inner_type;
6736                                 ether_type = rte_be_to_cpu_16(ether_type);
6737                         } else {
6738                                 ether_type = 0;
6739                         }
6740                         /* Store outer VLAN mask for of_push_vlan action. */
6741                         if (!tunnel)
6742                                 vlan_m = items->mask;
6743                         break;
6744                 case RTE_FLOW_ITEM_TYPE_IPV4:
6745                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6746                                                   &item_flags, &tunnel);
6747                         ret = flow_dv_validate_item_ipv4(items, item_flags,
6748                                                          last_item, ether_type,
6749                                                          error);
6750                         if (ret < 0)
6751                                 return ret;
6752                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
6753                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
6754                         if (items->mask != NULL &&
6755                             ((const struct rte_flow_item_ipv4 *)
6756                              items->mask)->hdr.next_proto_id) {
6757                                 next_protocol =
6758                                         ((const struct rte_flow_item_ipv4 *)
6759                                          (items->spec))->hdr.next_proto_id;
6760                                 next_protocol &=
6761                                         ((const struct rte_flow_item_ipv4 *)
6762                                          (items->mask))->hdr.next_proto_id;
6763                         } else {
6764                                 /* Reset for inner layer. */
6765                                 next_protocol = 0xff;
6766                         }
6767                         break;
6768                 case RTE_FLOW_ITEM_TYPE_IPV6:
6769                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6770                                                   &item_flags, &tunnel);
6771                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
6772                                                            last_item,
6773                                                            ether_type,
6774                                                            &nic_ipv6_mask,
6775                                                            error);
6776                         if (ret < 0)
6777                                 return ret;
6778                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
6779                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
6780                         if (items->mask != NULL &&
6781                             ((const struct rte_flow_item_ipv6 *)
6782                              items->mask)->hdr.proto) {
6783                                 item_ipv6_proto =
6784                                         ((const struct rte_flow_item_ipv6 *)
6785                                          items->spec)->hdr.proto;
6786                                 next_protocol =
6787                                         ((const struct rte_flow_item_ipv6 *)
6788                                          items->spec)->hdr.proto;
6789                                 next_protocol &=
6790                                         ((const struct rte_flow_item_ipv6 *)
6791                                          items->mask)->hdr.proto;
6792                         } else {
6793                                 /* Reset for inner layer. */
6794                                 next_protocol = 0xff;
6795                         }
6796                         break;
6797                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
6798                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
6799                                                                   item_flags,
6800                                                                   error);
6801                         if (ret < 0)
6802                                 return ret;
6803                         last_item = tunnel ?
6804                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
6805                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
6806                         if (items->mask != NULL &&
6807                             ((const struct rte_flow_item_ipv6_frag_ext *)
6808                              items->mask)->hdr.next_header) {
6809                                 next_protocol =
6810                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6811                                  items->spec)->hdr.next_header;
6812                                 next_protocol &=
6813                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6814                                  items->mask)->hdr.next_header;
6815                         } else {
6816                                 /* Reset for inner layer. */
6817                                 next_protocol = 0xff;
6818                         }
6819                         break;
6820                 case RTE_FLOW_ITEM_TYPE_TCP:
6821                         ret = mlx5_flow_validate_item_tcp
6822                                                 (items, item_flags,
6823                                                  next_protocol,
6824                                                  &nic_tcp_mask,
6825                                                  error);
6826                         if (ret < 0)
6827                                 return ret;
6828                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
6829                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
6830                         break;
6831                 case RTE_FLOW_ITEM_TYPE_UDP:
6832                         ret = mlx5_flow_validate_item_udp(items, item_flags,
6833                                                           next_protocol,
6834                                                           error);
6835                         if (ret < 0)
6836                                 return ret;
6837                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
6838                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
6839                         break;
6840                 case RTE_FLOW_ITEM_TYPE_GRE:
6841                         ret = mlx5_flow_validate_item_gre(items, item_flags,
6842                                                           next_protocol, error);
6843                         if (ret < 0)
6844                                 return ret;
6845                         gre_item = items;
6846                         last_item = MLX5_FLOW_LAYER_GRE;
6847                         break;
6848                 case RTE_FLOW_ITEM_TYPE_NVGRE:
6849                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
6850                                                             next_protocol,
6851                                                             error);
6852                         if (ret < 0)
6853                                 return ret;
6854                         last_item = MLX5_FLOW_LAYER_NVGRE;
6855                         break;
6856                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
6857                         ret = mlx5_flow_validate_item_gre_key
6858                                 (items, item_flags, gre_item, error);
6859                         if (ret < 0)
6860                                 return ret;
6861                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
6862                         break;
6863                 case RTE_FLOW_ITEM_TYPE_VXLAN:
6864                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
6865                                                             error);
6866                         if (ret < 0)
6867                                 return ret;
6868                         last_item = MLX5_FLOW_LAYER_VXLAN;
6869                         break;
6870                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
6871                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
6872                                                                 item_flags, dev,
6873                                                                 error);
6874                         if (ret < 0)
6875                                 return ret;
6876                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
6877                         break;
6878                 case RTE_FLOW_ITEM_TYPE_GENEVE:
6879                         ret = mlx5_flow_validate_item_geneve(items,
6880                                                              item_flags, dev,
6881                                                              error);
6882                         if (ret < 0)
6883                                 return ret;
6884                         geneve_item = items;
6885                         last_item = MLX5_FLOW_LAYER_GENEVE;
6886                         break;
6887                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
6888                         ret = mlx5_flow_validate_item_geneve_opt(items,
6889                                                                  last_item,
6890                                                                  geneve_item,
6891                                                                  dev,
6892                                                                  error);
6893                         if (ret < 0)
6894                                 return ret;
6895                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
6896                         break;
6897                 case RTE_FLOW_ITEM_TYPE_MPLS:
6898                         ret = mlx5_flow_validate_item_mpls(dev, items,
6899                                                            item_flags,
6900                                                            last_item, error);
6901                         if (ret < 0)
6902                                 return ret;
6903                         last_item = MLX5_FLOW_LAYER_MPLS;
6904                         break;
6905
6906                 case RTE_FLOW_ITEM_TYPE_MARK:
6907                         ret = flow_dv_validate_item_mark(dev, items, attr,
6908                                                          error);
6909                         if (ret < 0)
6910                                 return ret;
6911                         last_item = MLX5_FLOW_ITEM_MARK;
6912                         break;
6913                 case RTE_FLOW_ITEM_TYPE_META:
6914                         ret = flow_dv_validate_item_meta(dev, items, attr,
6915                                                          error);
6916                         if (ret < 0)
6917                                 return ret;
6918                         last_item = MLX5_FLOW_ITEM_METADATA;
6919                         break;
6920                 case RTE_FLOW_ITEM_TYPE_ICMP:
6921                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
6922                                                            next_protocol,
6923                                                            error);
6924                         if (ret < 0)
6925                                 return ret;
6926                         last_item = MLX5_FLOW_LAYER_ICMP;
6927                         break;
6928                 case RTE_FLOW_ITEM_TYPE_ICMP6:
6929                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
6930                                                             next_protocol,
6931                                                             error);
6932                         if (ret < 0)
6933                                 return ret;
6934                         item_ipv6_proto = IPPROTO_ICMPV6;
6935                         last_item = MLX5_FLOW_LAYER_ICMP6;
6936                         break;
6937                 case RTE_FLOW_ITEM_TYPE_TAG:
6938                         ret = flow_dv_validate_item_tag(dev, items,
6939                                                         attr, error);
6940                         if (ret < 0)
6941                                 return ret;
6942                         last_item = MLX5_FLOW_ITEM_TAG;
6943                         break;
6944                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
6945                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
6946                         break;
6947                 case RTE_FLOW_ITEM_TYPE_GTP:
6948                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
6949                                                         error);
6950                         if (ret < 0)
6951                                 return ret;
6952                         gtp_item = items;
6953                         last_item = MLX5_FLOW_LAYER_GTP;
6954                         break;
6955                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
6956                         ret = flow_dv_validate_item_gtp_psc(items, last_item,
6957                                                             gtp_item, attr,
6958                                                             error);
6959                         if (ret < 0)
6960                                 return ret;
6961                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
6962                         break;
6963                 case RTE_FLOW_ITEM_TYPE_ECPRI:
6964                         /* Capacity will be checked in the translate stage. */
6965                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
6966                                                             last_item,
6967                                                             ether_type,
6968                                                             &nic_ecpri_mask,
6969                                                             error);
6970                         if (ret < 0)
6971                                 return ret;
6972                         last_item = MLX5_FLOW_LAYER_ECPRI;
6973                         break;
6974                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
6975                         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY)
6976                                 return rte_flow_error_set
6977                                         (error, ENOTSUP,
6978                                          RTE_FLOW_ERROR_TYPE_ITEM,
6979                                          NULL, "multiple integrity items not supported");
6980                         ret = flow_dv_validate_item_integrity(dev, rule_items,
6981                                                               items, error);
6982                         if (ret < 0)
6983                                 return ret;
6984                         last_item = MLX5_FLOW_ITEM_INTEGRITY;
6985                         break;
6986                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
6987                         ret = flow_dv_validate_item_aso_ct(dev, items,
6988                                                            &item_flags, error);
6989                         if (ret < 0)
6990                                 return ret;
6991                         break;
6992                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
6993                         /* tunnel offload item was processed before
6994                          * list it here as a supported type
6995                          */
6996                         break;
6997                 default:
6998                         return rte_flow_error_set(error, ENOTSUP,
6999                                                   RTE_FLOW_ERROR_TYPE_ITEM,
7000                                                   NULL, "item not supported");
7001                 }
7002                 item_flags |= last_item;
7003         }
7004         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7005                 int type = actions->type;
7006                 bool shared_count = false;
7007
7008                 if (!mlx5_flow_os_action_supported(type))
7009                         return rte_flow_error_set(error, ENOTSUP,
7010                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7011                                                   actions,
7012                                                   "action not supported");
7013                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
7014                         return rte_flow_error_set(error, ENOTSUP,
7015                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7016                                                   actions, "too many actions");
7017                 if (action_flags &
7018                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
7019                         return rte_flow_error_set(error, ENOTSUP,
7020                                 RTE_FLOW_ERROR_TYPE_ACTION,
7021                                 NULL, "meter action with policy "
7022                                 "must be the last action");
7023                 switch (type) {
7024                 case RTE_FLOW_ACTION_TYPE_VOID:
7025                         break;
7026                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7027                         ret = flow_dv_validate_action_port_id(dev,
7028                                                               action_flags,
7029                                                               actions,
7030                                                               attr,
7031                                                               error);
7032                         if (ret)
7033                                 return ret;
7034                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7035                         ++actions_n;
7036                         break;
7037                 case RTE_FLOW_ACTION_TYPE_FLAG:
7038                         ret = flow_dv_validate_action_flag(dev, action_flags,
7039                                                            attr, error);
7040                         if (ret < 0)
7041                                 return ret;
7042                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7043                                 /* Count all modify-header actions as one. */
7044                                 if (!(action_flags &
7045                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7046                                         ++actions_n;
7047                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
7048                                                 MLX5_FLOW_ACTION_MARK_EXT;
7049                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7050                                         modify_after_mirror = 1;
7051
7052                         } else {
7053                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
7054                                 ++actions_n;
7055                         }
7056                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7057                         break;
7058                 case RTE_FLOW_ACTION_TYPE_MARK:
7059                         ret = flow_dv_validate_action_mark(dev, actions,
7060                                                            action_flags,
7061                                                            attr, error);
7062                         if (ret < 0)
7063                                 return ret;
7064                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7065                                 /* Count all modify-header actions as one. */
7066                                 if (!(action_flags &
7067                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7068                                         ++actions_n;
7069                                 action_flags |= MLX5_FLOW_ACTION_MARK |
7070                                                 MLX5_FLOW_ACTION_MARK_EXT;
7071                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7072                                         modify_after_mirror = 1;
7073                         } else {
7074                                 action_flags |= MLX5_FLOW_ACTION_MARK;
7075                                 ++actions_n;
7076                         }
7077                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7078                         break;
7079                 case RTE_FLOW_ACTION_TYPE_SET_META:
7080                         ret = flow_dv_validate_action_set_meta(dev, actions,
7081                                                                action_flags,
7082                                                                attr, error);
7083                         if (ret < 0)
7084                                 return ret;
7085                         /* Count all modify-header actions as one action. */
7086                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7087                                 ++actions_n;
7088                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7089                                 modify_after_mirror = 1;
7090                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7091                         rw_act_num += MLX5_ACT_NUM_SET_META;
7092                         break;
7093                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7094                         ret = flow_dv_validate_action_set_tag(dev, actions,
7095                                                               action_flags,
7096                                                               attr, error);
7097                         if (ret < 0)
7098                                 return ret;
7099                         /* Count all modify-header actions as one action. */
7100                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7101                                 ++actions_n;
7102                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7103                                 modify_after_mirror = 1;
7104                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7105                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7106                         break;
7107                 case RTE_FLOW_ACTION_TYPE_DROP:
7108                         ret = mlx5_flow_validate_action_drop(action_flags,
7109                                                              attr, error);
7110                         if (ret < 0)
7111                                 return ret;
7112                         action_flags |= MLX5_FLOW_ACTION_DROP;
7113                         ++actions_n;
7114                         break;
7115                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7116                         ret = mlx5_flow_validate_action_queue(actions,
7117                                                               action_flags, dev,
7118                                                               attr, error);
7119                         if (ret < 0)
7120                                 return ret;
7121                         queue_index = ((const struct rte_flow_action_queue *)
7122                                                         (actions->conf))->index;
7123                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7124                         ++actions_n;
7125                         break;
7126                 case RTE_FLOW_ACTION_TYPE_RSS:
7127                         rss = actions->conf;
7128                         ret = mlx5_flow_validate_action_rss(actions,
7129                                                             action_flags, dev,
7130                                                             attr, item_flags,
7131                                                             error);
7132                         if (ret < 0)
7133                                 return ret;
7134                         if (rss && sample_rss &&
7135                             (sample_rss->level != rss->level ||
7136                             sample_rss->types != rss->types))
7137                                 return rte_flow_error_set(error, ENOTSUP,
7138                                         RTE_FLOW_ERROR_TYPE_ACTION,
7139                                         NULL,
7140                                         "Can't use the different RSS types "
7141                                         "or level in the same flow");
7142                         if (rss != NULL && rss->queue_num)
7143                                 queue_index = rss->queue[0];
7144                         action_flags |= MLX5_FLOW_ACTION_RSS;
7145                         ++actions_n;
7146                         break;
7147                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
7148                         ret =
7149                         mlx5_flow_validate_action_default_miss(action_flags,
7150                                         attr, error);
7151                         if (ret < 0)
7152                                 return ret;
7153                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
7154                         ++actions_n;
7155                         break;
7156                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
7157                 case RTE_FLOW_ACTION_TYPE_COUNT:
7158                         shared_count = is_shared_action_count(actions);
7159                         ret = flow_dv_validate_action_count(dev, shared_count,
7160                                                             action_flags,
7161                                                             error);
7162                         if (ret < 0)
7163                                 return ret;
7164                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7165                         ++actions_n;
7166                         break;
7167                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7168                         if (flow_dv_validate_action_pop_vlan(dev,
7169                                                              action_flags,
7170                                                              actions,
7171                                                              item_flags, attr,
7172                                                              error))
7173                                 return -rte_errno;
7174                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7175                                 modify_after_mirror = 1;
7176                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7177                         ++actions_n;
7178                         break;
7179                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
7180                         ret = flow_dv_validate_action_push_vlan(dev,
7181                                                                 action_flags,
7182                                                                 vlan_m,
7183                                                                 actions, attr,
7184                                                                 error);
7185                         if (ret < 0)
7186                                 return ret;
7187                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7188                                 modify_after_mirror = 1;
7189                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
7190                         ++actions_n;
7191                         break;
7192                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
7193                         ret = flow_dv_validate_action_set_vlan_pcp
7194                                                 (action_flags, actions, error);
7195                         if (ret < 0)
7196                                 return ret;
7197                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7198                                 modify_after_mirror = 1;
7199                         /* Count PCP with push_vlan command. */
7200                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
7201                         break;
7202                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
7203                         ret = flow_dv_validate_action_set_vlan_vid
7204                                                 (item_flags, action_flags,
7205                                                  actions, error);
7206                         if (ret < 0)
7207                                 return ret;
7208                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7209                                 modify_after_mirror = 1;
7210                         /* Count VID with push_vlan command. */
7211                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
7212                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
7213                         break;
7214                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
7215                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
7216                         ret = flow_dv_validate_action_l2_encap(dev,
7217                                                                action_flags,
7218                                                                actions, attr,
7219                                                                error);
7220                         if (ret < 0)
7221                                 return ret;
7222                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
7223                         ++actions_n;
7224                         break;
7225                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
7226                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
7227                         ret = flow_dv_validate_action_decap(dev, action_flags,
7228                                                             actions, item_flags,
7229                                                             attr, error);
7230                         if (ret < 0)
7231                                 return ret;
7232                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7233                                 modify_after_mirror = 1;
7234                         action_flags |= MLX5_FLOW_ACTION_DECAP;
7235                         ++actions_n;
7236                         break;
7237                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
7238                         ret = flow_dv_validate_action_raw_encap_decap
7239                                 (dev, NULL, actions->conf, attr, &action_flags,
7240                                  &actions_n, actions, item_flags, error);
7241                         if (ret < 0)
7242                                 return ret;
7243                         break;
7244                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
7245                         decap = actions->conf;
7246                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
7247                                 ;
7248                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
7249                                 encap = NULL;
7250                                 actions--;
7251                         } else {
7252                                 encap = actions->conf;
7253                         }
7254                         ret = flow_dv_validate_action_raw_encap_decap
7255                                            (dev,
7256                                             decap ? decap : &empty_decap, encap,
7257                                             attr, &action_flags, &actions_n,
7258                                             actions, item_flags, error);
7259                         if (ret < 0)
7260                                 return ret;
7261                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7262                             (action_flags & MLX5_FLOW_ACTION_DECAP))
7263                                 modify_after_mirror = 1;
7264                         break;
7265                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
7266                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
7267                         ret = flow_dv_validate_action_modify_mac(action_flags,
7268                                                                  actions,
7269                                                                  item_flags,
7270                                                                  error);
7271                         if (ret < 0)
7272                                 return ret;
7273                         /* Count all modify-header actions as one action. */
7274                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7275                                 ++actions_n;
7276                         action_flags |= actions->type ==
7277                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
7278                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
7279                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
7280                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7281                                 modify_after_mirror = 1;
7282                         /*
7283                          * Even if the source and destination MAC addresses have
7284                          * overlap in the header with 4B alignment, the convert
7285                          * function will handle them separately and 4 SW actions
7286                          * will be created. And 2 actions will be added each
7287                          * time no matter how many bytes of address will be set.
7288                          */
7289                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
7290                         break;
7291                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
7292                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
7293                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
7294                                                                   actions,
7295                                                                   item_flags,
7296                                                                   error);
7297                         if (ret < 0)
7298                                 return ret;
7299                         /* Count all modify-header actions as one action. */
7300                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7301                                 ++actions_n;
7302                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7303                                 modify_after_mirror = 1;
7304                         action_flags |= actions->type ==
7305                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
7306                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
7307                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
7308                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
7309                         break;
7310                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
7311                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
7312                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
7313                                                                   actions,
7314                                                                   item_flags,
7315                                                                   error);
7316                         if (ret < 0)
7317                                 return ret;
7318                         if (item_ipv6_proto == IPPROTO_ICMPV6)
7319                                 return rte_flow_error_set(error, ENOTSUP,
7320                                         RTE_FLOW_ERROR_TYPE_ACTION,
7321                                         actions,
7322                                         "Can't change header "
7323                                         "with ICMPv6 proto");
7324                         /* Count all modify-header actions as one action. */
7325                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7326                                 ++actions_n;
7327                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7328                                 modify_after_mirror = 1;
7329                         action_flags |= actions->type ==
7330                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
7331                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
7332                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
7333                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
7334                         break;
7335                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
7336                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
7337                         ret = flow_dv_validate_action_modify_tp(action_flags,
7338                                                                 actions,
7339                                                                 item_flags,
7340                                                                 error);
7341                         if (ret < 0)
7342                                 return ret;
7343                         /* Count all modify-header actions as one action. */
7344                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7345                                 ++actions_n;
7346                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7347                                 modify_after_mirror = 1;
7348                         action_flags |= actions->type ==
7349                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
7350                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
7351                                                 MLX5_FLOW_ACTION_SET_TP_DST;
7352                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
7353                         break;
7354                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
7355                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
7356                         ret = flow_dv_validate_action_modify_ttl(action_flags,
7357                                                                  actions,
7358                                                                  item_flags,
7359                                                                  error);
7360                         if (ret < 0)
7361                                 return ret;
7362                         /* Count all modify-header actions as one action. */
7363                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7364                                 ++actions_n;
7365                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7366                                 modify_after_mirror = 1;
7367                         action_flags |= actions->type ==
7368                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
7369                                                 MLX5_FLOW_ACTION_SET_TTL :
7370                                                 MLX5_FLOW_ACTION_DEC_TTL;
7371                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
7372                         break;
7373                 case RTE_FLOW_ACTION_TYPE_JUMP:
7374                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
7375                                                            action_flags,
7376                                                            attr, external,
7377                                                            error);
7378                         if (ret)
7379                                 return ret;
7380                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7381                             fdb_mirror_limit)
7382                                 return rte_flow_error_set(error, EINVAL,
7383                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7384                                                   NULL,
7385                                                   "sample and jump action combination is not supported");
7386                         ++actions_n;
7387                         action_flags |= MLX5_FLOW_ACTION_JUMP;
7388                         break;
7389                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
7390                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
7391                         ret = flow_dv_validate_action_modify_tcp_seq
7392                                                                 (action_flags,
7393                                                                  actions,
7394                                                                  item_flags,
7395                                                                  error);
7396                         if (ret < 0)
7397                                 return ret;
7398                         /* Count all modify-header actions as one action. */
7399                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7400                                 ++actions_n;
7401                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7402                                 modify_after_mirror = 1;
7403                         action_flags |= actions->type ==
7404                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
7405                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
7406                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
7407                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
7408                         break;
7409                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
7410                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
7411                         ret = flow_dv_validate_action_modify_tcp_ack
7412                                                                 (action_flags,
7413                                                                  actions,
7414                                                                  item_flags,
7415                                                                  error);
7416                         if (ret < 0)
7417                                 return ret;
7418                         /* Count all modify-header actions as one action. */
7419                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7420                                 ++actions_n;
7421                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7422                                 modify_after_mirror = 1;
7423                         action_flags |= actions->type ==
7424                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
7425                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
7426                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
7427                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
7428                         break;
7429                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7430                         break;
7431                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
7432                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7433                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7434                         break;
7435                 case RTE_FLOW_ACTION_TYPE_METER:
7436                         ret = mlx5_flow_validate_action_meter(dev,
7437                                                               action_flags,
7438                                                               actions, attr,
7439                                                               &def_policy,
7440                                                               error);
7441                         if (ret < 0)
7442                                 return ret;
7443                         action_flags |= MLX5_FLOW_ACTION_METER;
7444                         if (!def_policy)
7445                                 action_flags |=
7446                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
7447                         ++actions_n;
7448                         /* Meter action will add one more TAG action. */
7449                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7450                         break;
7451                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
7452                         if (!attr->transfer && !attr->group)
7453                                 return rte_flow_error_set(error, ENOTSUP,
7454                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7455                                                                            NULL,
7456                           "Shared ASO age action is not supported for group 0");
7457                         if (action_flags & MLX5_FLOW_ACTION_AGE)
7458                                 return rte_flow_error_set
7459                                                   (error, EINVAL,
7460                                                    RTE_FLOW_ERROR_TYPE_ACTION,
7461                                                    NULL,
7462                                                    "duplicate age actions set");
7463                         action_flags |= MLX5_FLOW_ACTION_AGE;
7464                         ++actions_n;
7465                         break;
7466                 case RTE_FLOW_ACTION_TYPE_AGE:
7467                         ret = flow_dv_validate_action_age(action_flags,
7468                                                           actions, dev,
7469                                                           error);
7470                         if (ret < 0)
7471                                 return ret;
7472                         /*
7473                          * Validate the regular AGE action (using counter)
7474                          * mutual exclusion with share counter actions.
7475                          */
7476                         if (!priv->sh->flow_hit_aso_en) {
7477                                 if (shared_count)
7478                                         return rte_flow_error_set
7479                                                 (error, EINVAL,
7480                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7481                                                 NULL,
7482                                                 "old age and shared count combination is not supported");
7483                                 if (sample_count)
7484                                         return rte_flow_error_set
7485                                                 (error, EINVAL,
7486                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7487                                                 NULL,
7488                                                 "old age action and count must be in the same sub flow");
7489                         }
7490                         action_flags |= MLX5_FLOW_ACTION_AGE;
7491                         ++actions_n;
7492                         break;
7493                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
7494                         ret = flow_dv_validate_action_modify_ipv4_dscp
7495                                                          (action_flags,
7496                                                           actions,
7497                                                           item_flags,
7498                                                           error);
7499                         if (ret < 0)
7500                                 return ret;
7501                         /* Count all modify-header actions as one action. */
7502                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7503                                 ++actions_n;
7504                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7505                                 modify_after_mirror = 1;
7506                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
7507                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7508                         break;
7509                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
7510                         ret = flow_dv_validate_action_modify_ipv6_dscp
7511                                                                 (action_flags,
7512                                                                  actions,
7513                                                                  item_flags,
7514                                                                  error);
7515                         if (ret < 0)
7516                                 return ret;
7517                         /* Count all modify-header actions as one action. */
7518                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7519                                 ++actions_n;
7520                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7521                                 modify_after_mirror = 1;
7522                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
7523                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7524                         break;
7525                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
7526                         ret = flow_dv_validate_action_sample(&action_flags,
7527                                                              actions, dev,
7528                                                              attr, item_flags,
7529                                                              rss, &sample_rss,
7530                                                              &sample_count,
7531                                                              &fdb_mirror_limit,
7532                                                              error);
7533                         if (ret < 0)
7534                                 return ret;
7535                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
7536                         ++actions_n;
7537                         break;
7538                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
7539                         ret = flow_dv_validate_action_modify_field(dev,
7540                                                                    action_flags,
7541                                                                    actions,
7542                                                                    attr,
7543                                                                    error);
7544                         if (ret < 0)
7545                                 return ret;
7546                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7547                                 modify_after_mirror = 1;
7548                         /* Count all modify-header actions as one action. */
7549                         if (!(action_flags & MLX5_FLOW_ACTION_MODIFY_FIELD))
7550                                 ++actions_n;
7551                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
7552                         rw_act_num += ret;
7553                         break;
7554                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
7555                         ret = flow_dv_validate_action_aso_ct(dev, action_flags,
7556                                                              item_flags, attr,
7557                                                              error);
7558                         if (ret < 0)
7559                                 return ret;
7560                         action_flags |= MLX5_FLOW_ACTION_CT;
7561                         break;
7562                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
7563                         /* tunnel offload action was processed before
7564                          * list it here as a supported type
7565                          */
7566                         break;
7567                 default:
7568                         return rte_flow_error_set(error, ENOTSUP,
7569                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7570                                                   actions,
7571                                                   "action not supported");
7572                 }
7573         }
7574         /*
7575          * Validate actions in flow rules
7576          * - Explicit decap action is prohibited by the tunnel offload API.
7577          * - Drop action in tunnel steer rule is prohibited by the API.
7578          * - Application cannot use MARK action because it's value can mask
7579          *   tunnel default miss nitification.
7580          * - JUMP in tunnel match rule has no support in current PMD
7581          *   implementation.
7582          * - TAG & META are reserved for future uses.
7583          */
7584         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
7585                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
7586                                             MLX5_FLOW_ACTION_MARK     |
7587                                             MLX5_FLOW_ACTION_SET_TAG  |
7588                                             MLX5_FLOW_ACTION_SET_META |
7589                                             MLX5_FLOW_ACTION_DROP;
7590
7591                 if (action_flags & bad_actions_mask)
7592                         return rte_flow_error_set
7593                                         (error, EINVAL,
7594                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7595                                         "Invalid RTE action in tunnel "
7596                                         "set decap rule");
7597                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
7598                         return rte_flow_error_set
7599                                         (error, EINVAL,
7600                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7601                                         "tunnel set decap rule must terminate "
7602                                         "with JUMP");
7603                 if (!attr->ingress)
7604                         return rte_flow_error_set
7605                                         (error, EINVAL,
7606                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7607                                         "tunnel flows for ingress traffic only");
7608         }
7609         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
7610                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
7611                                             MLX5_FLOW_ACTION_MARK    |
7612                                             MLX5_FLOW_ACTION_SET_TAG |
7613                                             MLX5_FLOW_ACTION_SET_META;
7614
7615                 if (action_flags & bad_actions_mask)
7616                         return rte_flow_error_set
7617                                         (error, EINVAL,
7618                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7619                                         "Invalid RTE action in tunnel "
7620                                         "set match rule");
7621         }
7622         /*
7623          * Validate the drop action mutual exclusion with other actions.
7624          * Drop action is mutually-exclusive with any other action, except for
7625          * Count action.
7626          * Drop action compatibility with tunnel offload was already validated.
7627          */
7628         if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH |
7629                             MLX5_FLOW_ACTION_TUNNEL_MATCH));
7630         else if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
7631             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
7632                 return rte_flow_error_set(error, EINVAL,
7633                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7634                                           "Drop action is mutually-exclusive "
7635                                           "with any other action, except for "
7636                                           "Count action");
7637         /* Eswitch has few restrictions on using items and actions */
7638         if (attr->transfer) {
7639                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7640                     action_flags & MLX5_FLOW_ACTION_FLAG)
7641                         return rte_flow_error_set(error, ENOTSUP,
7642                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7643                                                   NULL,
7644                                                   "unsupported action FLAG");
7645                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7646                     action_flags & MLX5_FLOW_ACTION_MARK)
7647                         return rte_flow_error_set(error, ENOTSUP,
7648                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7649                                                   NULL,
7650                                                   "unsupported action MARK");
7651                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
7652                         return rte_flow_error_set(error, ENOTSUP,
7653                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7654                                                   NULL,
7655                                                   "unsupported action QUEUE");
7656                 if (action_flags & MLX5_FLOW_ACTION_RSS)
7657                         return rte_flow_error_set(error, ENOTSUP,
7658                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7659                                                   NULL,
7660                                                   "unsupported action RSS");
7661                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
7662                         return rte_flow_error_set(error, EINVAL,
7663                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7664                                                   actions,
7665                                                   "no fate action is found");
7666         } else {
7667                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
7668                         return rte_flow_error_set(error, EINVAL,
7669                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7670                                                   actions,
7671                                                   "no fate action is found");
7672         }
7673         /*
7674          * Continue validation for Xcap and VLAN actions.
7675          * If hairpin is working in explicit TX rule mode, there is no actions
7676          * splitting and the validation of hairpin ingress flow should be the
7677          * same as other standard flows.
7678          */
7679         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
7680                              MLX5_FLOW_VLAN_ACTIONS)) &&
7681             (queue_index == 0xFFFF ||
7682              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN ||
7683              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
7684              conf->tx_explicit != 0))) {
7685                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
7686                     MLX5_FLOW_XCAP_ACTIONS)
7687                         return rte_flow_error_set(error, ENOTSUP,
7688                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7689                                                   NULL, "encap and decap "
7690                                                   "combination aren't supported");
7691                 if (!attr->transfer && attr->ingress) {
7692                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7693                                 return rte_flow_error_set
7694                                                 (error, ENOTSUP,
7695                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7696                                                  NULL, "encap is not supported"
7697                                                  " for ingress traffic");
7698                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7699                                 return rte_flow_error_set
7700                                                 (error, ENOTSUP,
7701                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7702                                                  NULL, "push VLAN action not "
7703                                                  "supported for ingress");
7704                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
7705                                         MLX5_FLOW_VLAN_ACTIONS)
7706                                 return rte_flow_error_set
7707                                                 (error, ENOTSUP,
7708                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7709                                                  NULL, "no support for "
7710                                                  "multiple VLAN actions");
7711                 }
7712         }
7713         if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) {
7714                 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS &
7715                         ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) &&
7716                         attr->ingress)
7717                         return rte_flow_error_set
7718                                 (error, ENOTSUP,
7719                                 RTE_FLOW_ERROR_TYPE_ACTION,
7720                                 NULL, "fate action not supported for "
7721                                 "meter with policy");
7722                 if (attr->egress) {
7723                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
7724                                 return rte_flow_error_set
7725                                         (error, ENOTSUP,
7726                                         RTE_FLOW_ERROR_TYPE_ACTION,
7727                                         NULL, "modify header action in egress "
7728                                         "cannot be done before meter action");
7729                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7730                                 return rte_flow_error_set
7731                                         (error, ENOTSUP,
7732                                         RTE_FLOW_ERROR_TYPE_ACTION,
7733                                         NULL, "encap action in egress "
7734                                         "cannot be done before meter action");
7735                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7736                                 return rte_flow_error_set
7737                                         (error, ENOTSUP,
7738                                         RTE_FLOW_ERROR_TYPE_ACTION,
7739                                         NULL, "push vlan action in egress "
7740                                         "cannot be done before meter action");
7741                 }
7742         }
7743         /*
7744          * Hairpin flow will add one more TAG action in TX implicit mode.
7745          * In TX explicit mode, there will be no hairpin flow ID.
7746          */
7747         if (hairpin > 0)
7748                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7749         /* extra metadata enabled: one more TAG action will be add. */
7750         if (dev_conf->dv_flow_en &&
7751             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
7752             mlx5_flow_ext_mreg_supported(dev))
7753                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7754         if (rw_act_num >
7755                         flow_dv_modify_hdr_action_max(dev, is_root)) {
7756                 return rte_flow_error_set(error, ENOTSUP,
7757                                           RTE_FLOW_ERROR_TYPE_ACTION,
7758                                           NULL, "too many header modify"
7759                                           " actions to support");
7760         }
7761         /* Eswitch egress mirror and modify flow has limitation on CX5 */
7762         if (fdb_mirror_limit && modify_after_mirror)
7763                 return rte_flow_error_set(error, EINVAL,
7764                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7765                                 "sample before modify action is not supported");
7766         return 0;
7767 }
7768
7769 /**
7770  * Internal preparation function. Allocates the DV flow size,
7771  * this size is constant.
7772  *
7773  * @param[in] dev
7774  *   Pointer to the rte_eth_dev structure.
7775  * @param[in] attr
7776  *   Pointer to the flow attributes.
7777  * @param[in] items
7778  *   Pointer to the list of items.
7779  * @param[in] actions
7780  *   Pointer to the list of actions.
7781  * @param[out] error
7782  *   Pointer to the error structure.
7783  *
7784  * @return
7785  *   Pointer to mlx5_flow object on success,
7786  *   otherwise NULL and rte_errno is set.
7787  */
7788 static struct mlx5_flow *
7789 flow_dv_prepare(struct rte_eth_dev *dev,
7790                 const struct rte_flow_attr *attr __rte_unused,
7791                 const struct rte_flow_item items[] __rte_unused,
7792                 const struct rte_flow_action actions[] __rte_unused,
7793                 struct rte_flow_error *error)
7794 {
7795         uint32_t handle_idx = 0;
7796         struct mlx5_flow *dev_flow;
7797         struct mlx5_flow_handle *dev_handle;
7798         struct mlx5_priv *priv = dev->data->dev_private;
7799         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
7800
7801         MLX5_ASSERT(wks);
7802         wks->skip_matcher_reg = 0;
7803         /* In case of corrupting the memory. */
7804         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
7805                 rte_flow_error_set(error, ENOSPC,
7806                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7807                                    "not free temporary device flow");
7808                 return NULL;
7809         }
7810         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
7811                                    &handle_idx);
7812         if (!dev_handle) {
7813                 rte_flow_error_set(error, ENOMEM,
7814                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7815                                    "not enough memory to create flow handle");
7816                 return NULL;
7817         }
7818         MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows));
7819         dev_flow = &wks->flows[wks->flow_idx++];
7820         memset(dev_flow, 0, sizeof(*dev_flow));
7821         dev_flow->handle = dev_handle;
7822         dev_flow->handle_idx = handle_idx;
7823         /*
7824          * In some old rdma-core releases, before continuing, a check of the
7825          * length of matching parameter will be done at first. It needs to use
7826          * the length without misc4 param. If the flow has misc4 support, then
7827          * the length needs to be adjusted accordingly. Each param member is
7828          * aligned with a 64B boundary naturally.
7829          */
7830         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
7831                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
7832         dev_flow->ingress = attr->ingress;
7833         dev_flow->dv.transfer = attr->transfer;
7834         return dev_flow;
7835 }
7836
7837 #ifdef RTE_LIBRTE_MLX5_DEBUG
7838 /**
7839  * Sanity check for match mask and value. Similar to check_valid_spec() in
7840  * kernel driver. If unmasked bit is present in value, it returns failure.
7841  *
7842  * @param match_mask
7843  *   pointer to match mask buffer.
7844  * @param match_value
7845  *   pointer to match value buffer.
7846  *
7847  * @return
7848  *   0 if valid, -EINVAL otherwise.
7849  */
7850 static int
7851 flow_dv_check_valid_spec(void *match_mask, void *match_value)
7852 {
7853         uint8_t *m = match_mask;
7854         uint8_t *v = match_value;
7855         unsigned int i;
7856
7857         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
7858                 if (v[i] & ~m[i]) {
7859                         DRV_LOG(ERR,
7860                                 "match_value differs from match_criteria"
7861                                 " %p[%u] != %p[%u]",
7862                                 match_value, i, match_mask, i);
7863                         return -EINVAL;
7864                 }
7865         }
7866         return 0;
7867 }
7868 #endif
7869
7870 /**
7871  * Add match of ip_version.
7872  *
7873  * @param[in] group
7874  *   Flow group.
7875  * @param[in] headers_v
7876  *   Values header pointer.
7877  * @param[in] headers_m
7878  *   Masks header pointer.
7879  * @param[in] ip_version
7880  *   The IP version to set.
7881  */
7882 static inline void
7883 flow_dv_set_match_ip_version(uint32_t group,
7884                              void *headers_v,
7885                              void *headers_m,
7886                              uint8_t ip_version)
7887 {
7888         if (group == 0)
7889                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
7890         else
7891                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
7892                          ip_version);
7893         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
7894         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
7895         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
7896 }
7897
7898 /**
7899  * Add Ethernet item to matcher and to the value.
7900  *
7901  * @param[in, out] matcher
7902  *   Flow matcher.
7903  * @param[in, out] key
7904  *   Flow matcher value.
7905  * @param[in] item
7906  *   Flow pattern to translate.
7907  * @param[in] inner
7908  *   Item is inner pattern.
7909  */
7910 static void
7911 flow_dv_translate_item_eth(void *matcher, void *key,
7912                            const struct rte_flow_item *item, int inner,
7913                            uint32_t group)
7914 {
7915         const struct rte_flow_item_eth *eth_m = item->mask;
7916         const struct rte_flow_item_eth *eth_v = item->spec;
7917         const struct rte_flow_item_eth nic_mask = {
7918                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
7919                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
7920                 .type = RTE_BE16(0xffff),
7921                 .has_vlan = 0,
7922         };
7923         void *hdrs_m;
7924         void *hdrs_v;
7925         char *l24_v;
7926         unsigned int i;
7927
7928         if (!eth_v)
7929                 return;
7930         if (!eth_m)
7931                 eth_m = &nic_mask;
7932         if (inner) {
7933                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
7934                                          inner_headers);
7935                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7936         } else {
7937                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
7938                                          outer_headers);
7939                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7940         }
7941         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
7942                &eth_m->dst, sizeof(eth_m->dst));
7943         /* The value must be in the range of the mask. */
7944         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
7945         for (i = 0; i < sizeof(eth_m->dst); ++i)
7946                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
7947         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
7948                &eth_m->src, sizeof(eth_m->src));
7949         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
7950         /* The value must be in the range of the mask. */
7951         for (i = 0; i < sizeof(eth_m->dst); ++i)
7952                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
7953         /*
7954          * HW supports match on one Ethertype, the Ethertype following the last
7955          * VLAN tag of the packet (see PRM).
7956          * Set match on ethertype only if ETH header is not followed by VLAN.
7957          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
7958          * ethertype, and use ip_version field instead.
7959          * eCPRI over Ether layer will use type value 0xAEFE.
7960          */
7961         if (eth_m->type == 0xFFFF) {
7962                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
7963                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
7964                 switch (eth_v->type) {
7965                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
7966                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
7967                         return;
7968                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
7969                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
7970                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
7971                         return;
7972                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
7973                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
7974                         return;
7975                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
7976                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
7977                         return;
7978                 default:
7979                         break;
7980                 }
7981         }
7982         if (eth_m->has_vlan) {
7983                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
7984                 if (eth_v->has_vlan) {
7985                         /*
7986                          * Here, when also has_more_vlan field in VLAN item is
7987                          * not set, only single-tagged packets will be matched.
7988                          */
7989                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
7990                         return;
7991                 }
7992         }
7993         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
7994                  rte_be_to_cpu_16(eth_m->type));
7995         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
7996         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
7997 }
7998
7999 /**
8000  * Add VLAN item to matcher and to the value.
8001  *
8002  * @param[in, out] dev_flow
8003  *   Flow descriptor.
8004  * @param[in, out] matcher
8005  *   Flow matcher.
8006  * @param[in, out] key
8007  *   Flow matcher value.
8008  * @param[in] item
8009  *   Flow pattern to translate.
8010  * @param[in] inner
8011  *   Item is inner pattern.
8012  */
8013 static void
8014 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
8015                             void *matcher, void *key,
8016                             const struct rte_flow_item *item,
8017                             int inner, uint32_t group)
8018 {
8019         const struct rte_flow_item_vlan *vlan_m = item->mask;
8020         const struct rte_flow_item_vlan *vlan_v = item->spec;
8021         void *hdrs_m;
8022         void *hdrs_v;
8023         uint16_t tci_m;
8024         uint16_t tci_v;
8025
8026         if (inner) {
8027                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8028                                          inner_headers);
8029                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8030         } else {
8031                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8032                                          outer_headers);
8033                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8034                 /*
8035                  * This is workaround, masks are not supported,
8036                  * and pre-validated.
8037                  */
8038                 if (vlan_v)
8039                         dev_flow->handle->vf_vlan.tag =
8040                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
8041         }
8042         /*
8043          * When VLAN item exists in flow, mark packet as tagged,
8044          * even if TCI is not specified.
8045          */
8046         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
8047                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8048                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8049         }
8050         if (!vlan_v)
8051                 return;
8052         if (!vlan_m)
8053                 vlan_m = &rte_flow_item_vlan_mask;
8054         tci_m = rte_be_to_cpu_16(vlan_m->tci);
8055         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
8056         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
8057         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
8058         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
8059         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
8060         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
8061         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
8062         /*
8063          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8064          * ethertype, and use ip_version field instead.
8065          */
8066         if (vlan_m->inner_type == 0xFFFF) {
8067                 switch (vlan_v->inner_type) {
8068                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8069                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8070                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8071                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8072                         return;
8073                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8074                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8075                         return;
8076                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8077                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8078                         return;
8079                 default:
8080                         break;
8081                 }
8082         }
8083         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
8084                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8085                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8086                 /* Only one vlan_tag bit can be set. */
8087                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8088                 return;
8089         }
8090         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8091                  rte_be_to_cpu_16(vlan_m->inner_type));
8092         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
8093                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
8094 }
8095
8096 /**
8097  * Add IPV4 item to matcher and to the value.
8098  *
8099  * @param[in, out] matcher
8100  *   Flow matcher.
8101  * @param[in, out] key
8102  *   Flow matcher value.
8103  * @param[in] item
8104  *   Flow pattern to translate.
8105  * @param[in] inner
8106  *   Item is inner pattern.
8107  * @param[in] group
8108  *   The group to insert the rule.
8109  */
8110 static void
8111 flow_dv_translate_item_ipv4(void *matcher, void *key,
8112                             const struct rte_flow_item *item,
8113                             int inner, uint32_t group)
8114 {
8115         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
8116         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
8117         const struct rte_flow_item_ipv4 nic_mask = {
8118                 .hdr = {
8119                         .src_addr = RTE_BE32(0xffffffff),
8120                         .dst_addr = RTE_BE32(0xffffffff),
8121                         .type_of_service = 0xff,
8122                         .next_proto_id = 0xff,
8123                         .time_to_live = 0xff,
8124                 },
8125         };
8126         void *headers_m;
8127         void *headers_v;
8128         char *l24_m;
8129         char *l24_v;
8130         uint8_t tos;
8131
8132         if (inner) {
8133                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8134                                          inner_headers);
8135                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8136         } else {
8137                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8138                                          outer_headers);
8139                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8140         }
8141         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
8142         if (!ipv4_v)
8143                 return;
8144         if (!ipv4_m)
8145                 ipv4_m = &nic_mask;
8146         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8147                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8148         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8149                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8150         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
8151         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
8152         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8153                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8154         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8155                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8156         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
8157         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
8158         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
8159         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
8160                  ipv4_m->hdr.type_of_service);
8161         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
8162         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
8163                  ipv4_m->hdr.type_of_service >> 2);
8164         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
8165         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8166                  ipv4_m->hdr.next_proto_id);
8167         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8168                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
8169         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8170                  ipv4_m->hdr.time_to_live);
8171         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8172                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
8173         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8174                  !!(ipv4_m->hdr.fragment_offset));
8175         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8176                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
8177 }
8178
8179 /**
8180  * Add IPV6 item to matcher and to the value.
8181  *
8182  * @param[in, out] matcher
8183  *   Flow matcher.
8184  * @param[in, out] key
8185  *   Flow matcher value.
8186  * @param[in] item
8187  *   Flow pattern to translate.
8188  * @param[in] inner
8189  *   Item is inner pattern.
8190  * @param[in] group
8191  *   The group to insert the rule.
8192  */
8193 static void
8194 flow_dv_translate_item_ipv6(void *matcher, void *key,
8195                             const struct rte_flow_item *item,
8196                             int inner, uint32_t group)
8197 {
8198         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
8199         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
8200         const struct rte_flow_item_ipv6 nic_mask = {
8201                 .hdr = {
8202                         .src_addr =
8203                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8204                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8205                         .dst_addr =
8206                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8207                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8208                         .vtc_flow = RTE_BE32(0xffffffff),
8209                         .proto = 0xff,
8210                         .hop_limits = 0xff,
8211                 },
8212         };
8213         void *headers_m;
8214         void *headers_v;
8215         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8216         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8217         char *l24_m;
8218         char *l24_v;
8219         uint32_t vtc_m;
8220         uint32_t vtc_v;
8221         int i;
8222         int size;
8223
8224         if (inner) {
8225                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8226                                          inner_headers);
8227                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8228         } else {
8229                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8230                                          outer_headers);
8231                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8232         }
8233         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
8234         if (!ipv6_v)
8235                 return;
8236         if (!ipv6_m)
8237                 ipv6_m = &nic_mask;
8238         size = sizeof(ipv6_m->hdr.dst_addr);
8239         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8240                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8241         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8242                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8243         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
8244         for (i = 0; i < size; ++i)
8245                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
8246         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8247                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8248         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8249                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8250         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
8251         for (i = 0; i < size; ++i)
8252                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
8253         /* TOS. */
8254         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
8255         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
8256         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
8257         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
8258         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
8259         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
8260         /* Label. */
8261         if (inner) {
8262                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
8263                          vtc_m);
8264                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
8265                          vtc_v);
8266         } else {
8267                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
8268                          vtc_m);
8269                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
8270                          vtc_v);
8271         }
8272         /* Protocol. */
8273         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8274                  ipv6_m->hdr.proto);
8275         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8276                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
8277         /* Hop limit. */
8278         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8279                  ipv6_m->hdr.hop_limits);
8280         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8281                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
8282         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8283                  !!(ipv6_m->has_frag_ext));
8284         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8285                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
8286 }
8287
8288 /**
8289  * Add IPV6 fragment extension item to matcher and to the value.
8290  *
8291  * @param[in, out] matcher
8292  *   Flow matcher.
8293  * @param[in, out] key
8294  *   Flow matcher value.
8295  * @param[in] item
8296  *   Flow pattern to translate.
8297  * @param[in] inner
8298  *   Item is inner pattern.
8299  */
8300 static void
8301 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
8302                                      const struct rte_flow_item *item,
8303                                      int inner)
8304 {
8305         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
8306         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
8307         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
8308                 .hdr = {
8309                         .next_header = 0xff,
8310                         .frag_data = RTE_BE16(0xffff),
8311                 },
8312         };
8313         void *headers_m;
8314         void *headers_v;
8315
8316         if (inner) {
8317                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8318                                          inner_headers);
8319                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8320         } else {
8321                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8322                                          outer_headers);
8323                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8324         }
8325         /* IPv6 fragment extension item exists, so packet is IP fragment. */
8326         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
8327         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
8328         if (!ipv6_frag_ext_v)
8329                 return;
8330         if (!ipv6_frag_ext_m)
8331                 ipv6_frag_ext_m = &nic_mask;
8332         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8333                  ipv6_frag_ext_m->hdr.next_header);
8334         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8335                  ipv6_frag_ext_v->hdr.next_header &
8336                  ipv6_frag_ext_m->hdr.next_header);
8337 }
8338
8339 /**
8340  * Add TCP item to matcher and to the value.
8341  *
8342  * @param[in, out] matcher
8343  *   Flow matcher.
8344  * @param[in, out] key
8345  *   Flow matcher value.
8346  * @param[in] item
8347  *   Flow pattern to translate.
8348  * @param[in] inner
8349  *   Item is inner pattern.
8350  */
8351 static void
8352 flow_dv_translate_item_tcp(void *matcher, void *key,
8353                            const struct rte_flow_item *item,
8354                            int inner)
8355 {
8356         const struct rte_flow_item_tcp *tcp_m = item->mask;
8357         const struct rte_flow_item_tcp *tcp_v = item->spec;
8358         void *headers_m;
8359         void *headers_v;
8360
8361         if (inner) {
8362                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8363                                          inner_headers);
8364                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8365         } else {
8366                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8367                                          outer_headers);
8368                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8369         }
8370         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8371         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
8372         if (!tcp_v)
8373                 return;
8374         if (!tcp_m)
8375                 tcp_m = &rte_flow_item_tcp_mask;
8376         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
8377                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
8378         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
8379                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
8380         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
8381                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
8382         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
8383                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
8384         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
8385                  tcp_m->hdr.tcp_flags);
8386         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
8387                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
8388 }
8389
8390 /**
8391  * Add UDP item to matcher and to the value.
8392  *
8393  * @param[in, out] matcher
8394  *   Flow matcher.
8395  * @param[in, out] key
8396  *   Flow matcher value.
8397  * @param[in] item
8398  *   Flow pattern to translate.
8399  * @param[in] inner
8400  *   Item is inner pattern.
8401  */
8402 static void
8403 flow_dv_translate_item_udp(void *matcher, void *key,
8404                            const struct rte_flow_item *item,
8405                            int inner)
8406 {
8407         const struct rte_flow_item_udp *udp_m = item->mask;
8408         const struct rte_flow_item_udp *udp_v = item->spec;
8409         void *headers_m;
8410         void *headers_v;
8411
8412         if (inner) {
8413                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8414                                          inner_headers);
8415                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8416         } else {
8417                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8418                                          outer_headers);
8419                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8420         }
8421         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8422         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
8423         if (!udp_v)
8424                 return;
8425         if (!udp_m)
8426                 udp_m = &rte_flow_item_udp_mask;
8427         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
8428                  rte_be_to_cpu_16(udp_m->hdr.src_port));
8429         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
8430                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
8431         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
8432                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
8433         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
8434                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
8435 }
8436
8437 /**
8438  * Add GRE optional Key item to matcher and to the value.
8439  *
8440  * @param[in, out] matcher
8441  *   Flow matcher.
8442  * @param[in, out] key
8443  *   Flow matcher value.
8444  * @param[in] item
8445  *   Flow pattern to translate.
8446  * @param[in] inner
8447  *   Item is inner pattern.
8448  */
8449 static void
8450 flow_dv_translate_item_gre_key(void *matcher, void *key,
8451                                    const struct rte_flow_item *item)
8452 {
8453         const rte_be32_t *key_m = item->mask;
8454         const rte_be32_t *key_v = item->spec;
8455         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8456         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8457         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
8458
8459         /* GRE K bit must be on and should already be validated */
8460         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
8461         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
8462         if (!key_v)
8463                 return;
8464         if (!key_m)
8465                 key_m = &gre_key_default_mask;
8466         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
8467                  rte_be_to_cpu_32(*key_m) >> 8);
8468         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
8469                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
8470         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
8471                  rte_be_to_cpu_32(*key_m) & 0xFF);
8472         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
8473                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
8474 }
8475
8476 /**
8477  * Add GRE item to matcher and to the value.
8478  *
8479  * @param[in, out] matcher
8480  *   Flow matcher.
8481  * @param[in, out] key
8482  *   Flow matcher value.
8483  * @param[in] item
8484  *   Flow pattern to translate.
8485  * @param[in] inner
8486  *   Item is inner pattern.
8487  */
8488 static void
8489 flow_dv_translate_item_gre(void *matcher, void *key,
8490                            const struct rte_flow_item *item,
8491                            int inner)
8492 {
8493         const struct rte_flow_item_gre *gre_m = item->mask;
8494         const struct rte_flow_item_gre *gre_v = item->spec;
8495         void *headers_m;
8496         void *headers_v;
8497         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8498         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8499         struct {
8500                 union {
8501                         __extension__
8502                         struct {
8503                                 uint16_t version:3;
8504                                 uint16_t rsvd0:9;
8505                                 uint16_t s_present:1;
8506                                 uint16_t k_present:1;
8507                                 uint16_t rsvd_bit1:1;
8508                                 uint16_t c_present:1;
8509                         };
8510                         uint16_t value;
8511                 };
8512         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
8513
8514         if (inner) {
8515                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8516                                          inner_headers);
8517                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8518         } else {
8519                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8520                                          outer_headers);
8521                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8522         }
8523         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8524         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
8525         if (!gre_v)
8526                 return;
8527         if (!gre_m)
8528                 gre_m = &rte_flow_item_gre_mask;
8529         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
8530                  rte_be_to_cpu_16(gre_m->protocol));
8531         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
8532                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
8533         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
8534         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
8535         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
8536                  gre_crks_rsvd0_ver_m.c_present);
8537         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
8538                  gre_crks_rsvd0_ver_v.c_present &
8539                  gre_crks_rsvd0_ver_m.c_present);
8540         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
8541                  gre_crks_rsvd0_ver_m.k_present);
8542         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
8543                  gre_crks_rsvd0_ver_v.k_present &
8544                  gre_crks_rsvd0_ver_m.k_present);
8545         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
8546                  gre_crks_rsvd0_ver_m.s_present);
8547         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
8548                  gre_crks_rsvd0_ver_v.s_present &
8549                  gre_crks_rsvd0_ver_m.s_present);
8550 }
8551
8552 /**
8553  * Add NVGRE item to matcher and to the value.
8554  *
8555  * @param[in, out] matcher
8556  *   Flow matcher.
8557  * @param[in, out] key
8558  *   Flow matcher value.
8559  * @param[in] item
8560  *   Flow pattern to translate.
8561  * @param[in] inner
8562  *   Item is inner pattern.
8563  */
8564 static void
8565 flow_dv_translate_item_nvgre(void *matcher, void *key,
8566                              const struct rte_flow_item *item,
8567                              int inner)
8568 {
8569         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
8570         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
8571         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8572         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8573         const char *tni_flow_id_m;
8574         const char *tni_flow_id_v;
8575         char *gre_key_m;
8576         char *gre_key_v;
8577         int size;
8578         int i;
8579
8580         /* For NVGRE, GRE header fields must be set with defined values. */
8581         const struct rte_flow_item_gre gre_spec = {
8582                 .c_rsvd0_ver = RTE_BE16(0x2000),
8583                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
8584         };
8585         const struct rte_flow_item_gre gre_mask = {
8586                 .c_rsvd0_ver = RTE_BE16(0xB000),
8587                 .protocol = RTE_BE16(UINT16_MAX),
8588         };
8589         const struct rte_flow_item gre_item = {
8590                 .spec = &gre_spec,
8591                 .mask = &gre_mask,
8592                 .last = NULL,
8593         };
8594         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
8595         if (!nvgre_v)
8596                 return;
8597         if (!nvgre_m)
8598                 nvgre_m = &rte_flow_item_nvgre_mask;
8599         tni_flow_id_m = (const char *)nvgre_m->tni;
8600         tni_flow_id_v = (const char *)nvgre_v->tni;
8601         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
8602         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
8603         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
8604         memcpy(gre_key_m, tni_flow_id_m, size);
8605         for (i = 0; i < size; ++i)
8606                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
8607 }
8608
8609 /**
8610  * Add VXLAN item to matcher and to the value.
8611  *
8612  * @param[in, out] matcher
8613  *   Flow matcher.
8614  * @param[in, out] key
8615  *   Flow matcher value.
8616  * @param[in] item
8617  *   Flow pattern to translate.
8618  * @param[in] inner
8619  *   Item is inner pattern.
8620  */
8621 static void
8622 flow_dv_translate_item_vxlan(void *matcher, void *key,
8623                              const struct rte_flow_item *item,
8624                              int inner)
8625 {
8626         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
8627         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
8628         void *headers_m;
8629         void *headers_v;
8630         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8631         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8632         char *vni_m;
8633         char *vni_v;
8634         uint16_t dport;
8635         int size;
8636         int i;
8637
8638         if (inner) {
8639                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8640                                          inner_headers);
8641                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8642         } else {
8643                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8644                                          outer_headers);
8645                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8646         }
8647         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8648                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8649         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8650                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8651                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8652         }
8653         if (!vxlan_v)
8654                 return;
8655         if (!vxlan_m)
8656                 vxlan_m = &rte_flow_item_vxlan_mask;
8657         size = sizeof(vxlan_m->vni);
8658         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
8659         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
8660         memcpy(vni_m, vxlan_m->vni, size);
8661         for (i = 0; i < size; ++i)
8662                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8663 }
8664
8665 /**
8666  * Add VXLAN-GPE item to matcher and to the value.
8667  *
8668  * @param[in, out] matcher
8669  *   Flow matcher.
8670  * @param[in, out] key
8671  *   Flow matcher value.
8672  * @param[in] item
8673  *   Flow pattern to translate.
8674  * @param[in] inner
8675  *   Item is inner pattern.
8676  */
8677
8678 static void
8679 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
8680                                  const struct rte_flow_item *item, int inner)
8681 {
8682         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
8683         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
8684         void *headers_m;
8685         void *headers_v;
8686         void *misc_m =
8687                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
8688         void *misc_v =
8689                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
8690         char *vni_m;
8691         char *vni_v;
8692         uint16_t dport;
8693         int size;
8694         int i;
8695         uint8_t flags_m = 0xff;
8696         uint8_t flags_v = 0xc;
8697
8698         if (inner) {
8699                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8700                                          inner_headers);
8701                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8702         } else {
8703                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8704                                          outer_headers);
8705                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8706         }
8707         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8708                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8709         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8710                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8711                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8712         }
8713         if (!vxlan_v)
8714                 return;
8715         if (!vxlan_m)
8716                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
8717         size = sizeof(vxlan_m->vni);
8718         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
8719         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
8720         memcpy(vni_m, vxlan_m->vni, size);
8721         for (i = 0; i < size; ++i)
8722                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8723         if (vxlan_m->flags) {
8724                 flags_m = vxlan_m->flags;
8725                 flags_v = vxlan_v->flags;
8726         }
8727         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
8728         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
8729         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
8730                  vxlan_m->protocol);
8731         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
8732                  vxlan_v->protocol);
8733 }
8734
8735 /**
8736  * Add Geneve item to matcher and to the value.
8737  *
8738  * @param[in, out] matcher
8739  *   Flow matcher.
8740  * @param[in, out] key
8741  *   Flow matcher value.
8742  * @param[in] item
8743  *   Flow pattern to translate.
8744  * @param[in] inner
8745  *   Item is inner pattern.
8746  */
8747
8748 static void
8749 flow_dv_translate_item_geneve(void *matcher, void *key,
8750                               const struct rte_flow_item *item, int inner)
8751 {
8752         const struct rte_flow_item_geneve *geneve_m = item->mask;
8753         const struct rte_flow_item_geneve *geneve_v = item->spec;
8754         void *headers_m;
8755         void *headers_v;
8756         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8757         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8758         uint16_t dport;
8759         uint16_t gbhdr_m;
8760         uint16_t gbhdr_v;
8761         char *vni_m;
8762         char *vni_v;
8763         size_t size, i;
8764
8765         if (inner) {
8766                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8767                                          inner_headers);
8768                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8769         } else {
8770                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8771                                          outer_headers);
8772                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8773         }
8774         dport = MLX5_UDP_PORT_GENEVE;
8775         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8776                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8777                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8778         }
8779         if (!geneve_v)
8780                 return;
8781         if (!geneve_m)
8782                 geneve_m = &rte_flow_item_geneve_mask;
8783         size = sizeof(geneve_m->vni);
8784         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
8785         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
8786         memcpy(vni_m, geneve_m->vni, size);
8787         for (i = 0; i < size; ++i)
8788                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
8789         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
8790                  rte_be_to_cpu_16(geneve_m->protocol));
8791         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
8792                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
8793         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
8794         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
8795         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
8796                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
8797         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
8798                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
8799         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
8800                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
8801         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
8802                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
8803                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
8804 }
8805
8806 /**
8807  * Create Geneve TLV option resource.
8808  *
8809  * @param dev[in, out]
8810  *   Pointer to rte_eth_dev structure.
8811  * @param[in, out] tag_be24
8812  *   Tag value in big endian then R-shift 8.
8813  * @parm[in, out] dev_flow
8814  *   Pointer to the dev_flow.
8815  * @param[out] error
8816  *   pointer to error structure.
8817  *
8818  * @return
8819  *   0 on success otherwise -errno and errno is set.
8820  */
8821
8822 int
8823 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
8824                                              const struct rte_flow_item *item,
8825                                              struct rte_flow_error *error)
8826 {
8827         struct mlx5_priv *priv = dev->data->dev_private;
8828         struct mlx5_dev_ctx_shared *sh = priv->sh;
8829         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
8830                         sh->geneve_tlv_option_resource;
8831         struct mlx5_devx_obj *obj;
8832         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
8833         int ret = 0;
8834
8835         if (!geneve_opt_v)
8836                 return -1;
8837         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
8838         if (geneve_opt_resource != NULL) {
8839                 if (geneve_opt_resource->option_class ==
8840                         geneve_opt_v->option_class &&
8841                         geneve_opt_resource->option_type ==
8842                         geneve_opt_v->option_type &&
8843                         geneve_opt_resource->length ==
8844                         geneve_opt_v->option_len) {
8845                         /* We already have GENVE TLV option obj allocated. */
8846                         __atomic_fetch_add(&geneve_opt_resource->refcnt, 1,
8847                                            __ATOMIC_RELAXED);
8848                 } else {
8849                         ret = rte_flow_error_set(error, ENOMEM,
8850                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8851                                 "Only one GENEVE TLV option supported");
8852                         goto exit;
8853                 }
8854         } else {
8855                 /* Create a GENEVE TLV object and resource. */
8856                 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->ctx,
8857                                 geneve_opt_v->option_class,
8858                                 geneve_opt_v->option_type,
8859                                 geneve_opt_v->option_len);
8860                 if (!obj) {
8861                         ret = rte_flow_error_set(error, ENODATA,
8862                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8863                                 "Failed to create GENEVE TLV Devx object");
8864                         goto exit;
8865                 }
8866                 sh->geneve_tlv_option_resource =
8867                                 mlx5_malloc(MLX5_MEM_ZERO,
8868                                                 sizeof(*geneve_opt_resource),
8869                                                 0, SOCKET_ID_ANY);
8870                 if (!sh->geneve_tlv_option_resource) {
8871                         claim_zero(mlx5_devx_cmd_destroy(obj));
8872                         ret = rte_flow_error_set(error, ENOMEM,
8873                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8874                                 "GENEVE TLV object memory allocation failed");
8875                         goto exit;
8876                 }
8877                 geneve_opt_resource = sh->geneve_tlv_option_resource;
8878                 geneve_opt_resource->obj = obj;
8879                 geneve_opt_resource->option_class = geneve_opt_v->option_class;
8880                 geneve_opt_resource->option_type = geneve_opt_v->option_type;
8881                 geneve_opt_resource->length = geneve_opt_v->option_len;
8882                 __atomic_store_n(&geneve_opt_resource->refcnt, 1,
8883                                 __ATOMIC_RELAXED);
8884         }
8885 exit:
8886         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
8887         return ret;
8888 }
8889
8890 /**
8891  * Add Geneve TLV option item to matcher.
8892  *
8893  * @param[in, out] dev
8894  *   Pointer to rte_eth_dev structure.
8895  * @param[in, out] matcher
8896  *   Flow matcher.
8897  * @param[in, out] key
8898  *   Flow matcher value.
8899  * @param[in] item
8900  *   Flow pattern to translate.
8901  * @param[out] error
8902  *   Pointer to error structure.
8903  */
8904 static int
8905 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher,
8906                                   void *key, const struct rte_flow_item *item,
8907                                   struct rte_flow_error *error)
8908 {
8909         const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask;
8910         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
8911         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8912         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8913         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
8914                         misc_parameters_3);
8915         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
8916         rte_be32_t opt_data_key = 0, opt_data_mask = 0;
8917         int ret = 0;
8918
8919         if (!geneve_opt_v)
8920                 return -1;
8921         if (!geneve_opt_m)
8922                 geneve_opt_m = &rte_flow_item_geneve_opt_mask;
8923         ret = flow_dev_geneve_tlv_option_resource_register(dev, item,
8924                                                            error);
8925         if (ret) {
8926                 DRV_LOG(ERR, "Failed to create geneve_tlv_obj");
8927                 return ret;
8928         }
8929         /*
8930          * Set the option length in GENEVE header if not requested.
8931          * The GENEVE TLV option length is expressed by the option length field
8932          * in the GENEVE header.
8933          * If the option length was not requested but the GENEVE TLV option item
8934          * is present we set the option length field implicitly.
8935          */
8936         if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) {
8937                 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
8938                          MLX5_GENEVE_OPTLEN_MASK);
8939                 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
8940                          geneve_opt_v->option_len + 1);
8941         }
8942         /* Set the data. */
8943         if (geneve_opt_v->data) {
8944                 memcpy(&opt_data_key, geneve_opt_v->data,
8945                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
8946                                 sizeof(opt_data_key)));
8947                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
8948                                 sizeof(opt_data_key));
8949                 memcpy(&opt_data_mask, geneve_opt_m->data,
8950                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
8951                                 sizeof(opt_data_mask)));
8952                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
8953                                 sizeof(opt_data_mask));
8954                 MLX5_SET(fte_match_set_misc3, misc3_m,
8955                                 geneve_tlv_option_0_data,
8956                                 rte_be_to_cpu_32(opt_data_mask));
8957                 MLX5_SET(fte_match_set_misc3, misc3_v,
8958                                 geneve_tlv_option_0_data,
8959                         rte_be_to_cpu_32(opt_data_key & opt_data_mask));
8960         }
8961         return ret;
8962 }
8963
8964 /**
8965  * Add MPLS item to matcher and to the value.
8966  *
8967  * @param[in, out] matcher
8968  *   Flow matcher.
8969  * @param[in, out] key
8970  *   Flow matcher value.
8971  * @param[in] item
8972  *   Flow pattern to translate.
8973  * @param[in] prev_layer
8974  *   The protocol layer indicated in previous item.
8975  * @param[in] inner
8976  *   Item is inner pattern.
8977  */
8978 static void
8979 flow_dv_translate_item_mpls(void *matcher, void *key,
8980                             const struct rte_flow_item *item,
8981                             uint64_t prev_layer,
8982                             int inner)
8983 {
8984         const uint32_t *in_mpls_m = item->mask;
8985         const uint32_t *in_mpls_v = item->spec;
8986         uint32_t *out_mpls_m = 0;
8987         uint32_t *out_mpls_v = 0;
8988         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8989         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8990         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
8991                                      misc_parameters_2);
8992         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
8993         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
8994         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8995
8996         switch (prev_layer) {
8997         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
8998                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
8999                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9000                          MLX5_UDP_PORT_MPLS);
9001                 break;
9002         case MLX5_FLOW_LAYER_GRE:
9003                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
9004                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
9005                          RTE_ETHER_TYPE_MPLS);
9006                 break;
9007         default:
9008                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
9009                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
9010                          IPPROTO_MPLS);
9011                 break;
9012         }
9013         if (!in_mpls_v)
9014                 return;
9015         if (!in_mpls_m)
9016                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
9017         switch (prev_layer) {
9018         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9019                 out_mpls_m =
9020                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9021                                                  outer_first_mpls_over_udp);
9022                 out_mpls_v =
9023                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9024                                                  outer_first_mpls_over_udp);
9025                 break;
9026         case MLX5_FLOW_LAYER_GRE:
9027                 out_mpls_m =
9028                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9029                                                  outer_first_mpls_over_gre);
9030                 out_mpls_v =
9031                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9032                                                  outer_first_mpls_over_gre);
9033                 break;
9034         default:
9035                 /* Inner MPLS not over GRE is not supported. */
9036                 if (!inner) {
9037                         out_mpls_m =
9038                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9039                                                          misc2_m,
9040                                                          outer_first_mpls);
9041                         out_mpls_v =
9042                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9043                                                          misc2_v,
9044                                                          outer_first_mpls);
9045                 }
9046                 break;
9047         }
9048         if (out_mpls_m && out_mpls_v) {
9049                 *out_mpls_m = *in_mpls_m;
9050                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
9051         }
9052 }
9053
9054 /**
9055  * Add metadata register item to matcher
9056  *
9057  * @param[in, out] matcher
9058  *   Flow matcher.
9059  * @param[in, out] key
9060  *   Flow matcher value.
9061  * @param[in] reg_type
9062  *   Type of device metadata register
9063  * @param[in] value
9064  *   Register value
9065  * @param[in] mask
9066  *   Register mask
9067  */
9068 static void
9069 flow_dv_match_meta_reg(void *matcher, void *key,
9070                        enum modify_reg reg_type,
9071                        uint32_t data, uint32_t mask)
9072 {
9073         void *misc2_m =
9074                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
9075         void *misc2_v =
9076                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9077         uint32_t temp;
9078
9079         data &= mask;
9080         switch (reg_type) {
9081         case REG_A:
9082                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
9083                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
9084                 break;
9085         case REG_B:
9086                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
9087                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
9088                 break;
9089         case REG_C_0:
9090                 /*
9091                  * The metadata register C0 field might be divided into
9092                  * source vport index and META item value, we should set
9093                  * this field according to specified mask, not as whole one.
9094                  */
9095                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
9096                 temp |= mask;
9097                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
9098                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
9099                 temp &= ~mask;
9100                 temp |= data;
9101                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
9102                 break;
9103         case REG_C_1:
9104                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
9105                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
9106                 break;
9107         case REG_C_2:
9108                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
9109                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
9110                 break;
9111         case REG_C_3:
9112                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
9113                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
9114                 break;
9115         case REG_C_4:
9116                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
9117                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
9118                 break;
9119         case REG_C_5:
9120                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
9121                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
9122                 break;
9123         case REG_C_6:
9124                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
9125                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
9126                 break;
9127         case REG_C_7:
9128                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
9129                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
9130                 break;
9131         default:
9132                 MLX5_ASSERT(false);
9133                 break;
9134         }
9135 }
9136
9137 /**
9138  * Add MARK item to matcher
9139  *
9140  * @param[in] dev
9141  *   The device to configure through.
9142  * @param[in, out] matcher
9143  *   Flow matcher.
9144  * @param[in, out] key
9145  *   Flow matcher value.
9146  * @param[in] item
9147  *   Flow pattern to translate.
9148  */
9149 static void
9150 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
9151                             void *matcher, void *key,
9152                             const struct rte_flow_item *item)
9153 {
9154         struct mlx5_priv *priv = dev->data->dev_private;
9155         const struct rte_flow_item_mark *mark;
9156         uint32_t value;
9157         uint32_t mask;
9158
9159         mark = item->mask ? (const void *)item->mask :
9160                             &rte_flow_item_mark_mask;
9161         mask = mark->id & priv->sh->dv_mark_mask;
9162         mark = (const void *)item->spec;
9163         MLX5_ASSERT(mark);
9164         value = mark->id & priv->sh->dv_mark_mask & mask;
9165         if (mask) {
9166                 enum modify_reg reg;
9167
9168                 /* Get the metadata register index for the mark. */
9169                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
9170                 MLX5_ASSERT(reg > 0);
9171                 if (reg == REG_C_0) {
9172                         struct mlx5_priv *priv = dev->data->dev_private;
9173                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9174                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9175
9176                         mask &= msk_c0;
9177                         mask <<= shl_c0;
9178                         value <<= shl_c0;
9179                 }
9180                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9181         }
9182 }
9183
9184 /**
9185  * Add META item to matcher
9186  *
9187  * @param[in] dev
9188  *   The devich to configure through.
9189  * @param[in, out] matcher
9190  *   Flow matcher.
9191  * @param[in, out] key
9192  *   Flow matcher value.
9193  * @param[in] attr
9194  *   Attributes of flow that includes this item.
9195  * @param[in] item
9196  *   Flow pattern to translate.
9197  */
9198 static void
9199 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
9200                             void *matcher, void *key,
9201                             const struct rte_flow_attr *attr,
9202                             const struct rte_flow_item *item)
9203 {
9204         const struct rte_flow_item_meta *meta_m;
9205         const struct rte_flow_item_meta *meta_v;
9206
9207         meta_m = (const void *)item->mask;
9208         if (!meta_m)
9209                 meta_m = &rte_flow_item_meta_mask;
9210         meta_v = (const void *)item->spec;
9211         if (meta_v) {
9212                 int reg;
9213                 uint32_t value = meta_v->data;
9214                 uint32_t mask = meta_m->data;
9215
9216                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
9217                 if (reg < 0)
9218                         return;
9219                 MLX5_ASSERT(reg != REG_NON);
9220                 /*
9221                  * In datapath code there is no endianness
9222                  * coversions for perfromance reasons, all
9223                  * pattern conversions are done in rte_flow.
9224                  */
9225                 value = rte_cpu_to_be_32(value);
9226                 mask = rte_cpu_to_be_32(mask);
9227                 if (reg == REG_C_0) {
9228                         struct mlx5_priv *priv = dev->data->dev_private;
9229                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9230                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9231 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
9232                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
9233
9234                         value >>= shr_c0;
9235                         mask >>= shr_c0;
9236 #endif
9237                         value <<= shl_c0;
9238                         mask <<= shl_c0;
9239                         MLX5_ASSERT(msk_c0);
9240                         MLX5_ASSERT(!(~msk_c0 & mask));
9241                 }
9242                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9243         }
9244 }
9245
9246 /**
9247  * Add vport metadata Reg C0 item to matcher
9248  *
9249  * @param[in, out] matcher
9250  *   Flow matcher.
9251  * @param[in, out] key
9252  *   Flow matcher value.
9253  * @param[in] reg
9254  *   Flow pattern to translate.
9255  */
9256 static void
9257 flow_dv_translate_item_meta_vport(void *matcher, void *key,
9258                                   uint32_t value, uint32_t mask)
9259 {
9260         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
9261 }
9262
9263 /**
9264  * Add tag item to matcher
9265  *
9266  * @param[in] dev
9267  *   The devich to configure through.
9268  * @param[in, out] matcher
9269  *   Flow matcher.
9270  * @param[in, out] key
9271  *   Flow matcher value.
9272  * @param[in] item
9273  *   Flow pattern to translate.
9274  */
9275 static void
9276 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
9277                                 void *matcher, void *key,
9278                                 const struct rte_flow_item *item)
9279 {
9280         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
9281         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
9282         uint32_t mask, value;
9283
9284         MLX5_ASSERT(tag_v);
9285         value = tag_v->data;
9286         mask = tag_m ? tag_m->data : UINT32_MAX;
9287         if (tag_v->id == REG_C_0) {
9288                 struct mlx5_priv *priv = dev->data->dev_private;
9289                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9290                 uint32_t shl_c0 = rte_bsf32(msk_c0);
9291
9292                 mask &= msk_c0;
9293                 mask <<= shl_c0;
9294                 value <<= shl_c0;
9295         }
9296         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
9297 }
9298
9299 /**
9300  * Add TAG item to matcher
9301  *
9302  * @param[in] dev
9303  *   The devich to configure through.
9304  * @param[in, out] matcher
9305  *   Flow matcher.
9306  * @param[in, out] key
9307  *   Flow matcher value.
9308  * @param[in] item
9309  *   Flow pattern to translate.
9310  */
9311 static void
9312 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
9313                            void *matcher, void *key,
9314                            const struct rte_flow_item *item)
9315 {
9316         const struct rte_flow_item_tag *tag_v = item->spec;
9317         const struct rte_flow_item_tag *tag_m = item->mask;
9318         enum modify_reg reg;
9319
9320         MLX5_ASSERT(tag_v);
9321         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
9322         /* Get the metadata register index for the tag. */
9323         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
9324         MLX5_ASSERT(reg > 0);
9325         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
9326 }
9327
9328 /**
9329  * Add source vport match to the specified matcher.
9330  *
9331  * @param[in, out] matcher
9332  *   Flow matcher.
9333  * @param[in, out] key
9334  *   Flow matcher value.
9335  * @param[in] port
9336  *   Source vport value to match
9337  * @param[in] mask
9338  *   Mask
9339  */
9340 static void
9341 flow_dv_translate_item_source_vport(void *matcher, void *key,
9342                                     int16_t port, uint16_t mask)
9343 {
9344         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9345         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9346
9347         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
9348         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
9349 }
9350
9351 /**
9352  * Translate port-id item to eswitch match on  port-id.
9353  *
9354  * @param[in] dev
9355  *   The devich to configure through.
9356  * @param[in, out] matcher
9357  *   Flow matcher.
9358  * @param[in, out] key
9359  *   Flow matcher value.
9360  * @param[in] item
9361  *   Flow pattern to translate.
9362  * @param[in]
9363  *   Flow attributes.
9364  *
9365  * @return
9366  *   0 on success, a negative errno value otherwise.
9367  */
9368 static int
9369 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
9370                                void *key, const struct rte_flow_item *item,
9371                                const struct rte_flow_attr *attr)
9372 {
9373         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
9374         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
9375         struct mlx5_priv *priv;
9376         uint16_t mask, id;
9377
9378         mask = pid_m ? pid_m->id : 0xffff;
9379         id = pid_v ? pid_v->id : dev->data->port_id;
9380         priv = mlx5_port_to_eswitch_info(id, item == NULL);
9381         if (!priv)
9382                 return -rte_errno;
9383         /*
9384          * Translate to vport field or to metadata, depending on mode.
9385          * Kernel can use either misc.source_port or half of C0 metadata
9386          * register.
9387          */
9388         if (priv->vport_meta_mask) {
9389                 /*
9390                  * Provide the hint for SW steering library
9391                  * to insert the flow into ingress domain and
9392                  * save the extra vport match.
9393                  */
9394                 if (mask == 0xffff && priv->vport_id == 0xffff &&
9395                     priv->pf_bond < 0 && attr->transfer)
9396                         flow_dv_translate_item_source_vport
9397                                 (matcher, key, priv->vport_id, mask);
9398                 /*
9399                  * We should always set the vport metadata register,
9400                  * otherwise the SW steering library can drop
9401                  * the rule if wire vport metadata value is not zero,
9402                  * it depends on kernel configuration.
9403                  */
9404                 flow_dv_translate_item_meta_vport(matcher, key,
9405                                                   priv->vport_meta_tag,
9406                                                   priv->vport_meta_mask);
9407         } else {
9408                 flow_dv_translate_item_source_vport(matcher, key,
9409                                                     priv->vport_id, mask);
9410         }
9411         return 0;
9412 }
9413
9414 /**
9415  * Add ICMP6 item to matcher and to the value.
9416  *
9417  * @param[in, out] matcher
9418  *   Flow matcher.
9419  * @param[in, out] key
9420  *   Flow matcher value.
9421  * @param[in] item
9422  *   Flow pattern to translate.
9423  * @param[in] inner
9424  *   Item is inner pattern.
9425  */
9426 static void
9427 flow_dv_translate_item_icmp6(void *matcher, void *key,
9428                               const struct rte_flow_item *item,
9429                               int inner)
9430 {
9431         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
9432         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
9433         void *headers_m;
9434         void *headers_v;
9435         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9436                                      misc_parameters_3);
9437         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9438         if (inner) {
9439                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9440                                          inner_headers);
9441                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9442         } else {
9443                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9444                                          outer_headers);
9445                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9446         }
9447         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9448         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
9449         if (!icmp6_v)
9450                 return;
9451         if (!icmp6_m)
9452                 icmp6_m = &rte_flow_item_icmp6_mask;
9453         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
9454         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
9455                  icmp6_v->type & icmp6_m->type);
9456         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
9457         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
9458                  icmp6_v->code & icmp6_m->code);
9459 }
9460
9461 /**
9462  * Add ICMP item to matcher and to the value.
9463  *
9464  * @param[in, out] matcher
9465  *   Flow matcher.
9466  * @param[in, out] key
9467  *   Flow matcher value.
9468  * @param[in] item
9469  *   Flow pattern to translate.
9470  * @param[in] inner
9471  *   Item is inner pattern.
9472  */
9473 static void
9474 flow_dv_translate_item_icmp(void *matcher, void *key,
9475                             const struct rte_flow_item *item,
9476                             int inner)
9477 {
9478         const struct rte_flow_item_icmp *icmp_m = item->mask;
9479         const struct rte_flow_item_icmp *icmp_v = item->spec;
9480         uint32_t icmp_header_data_m = 0;
9481         uint32_t icmp_header_data_v = 0;
9482         void *headers_m;
9483         void *headers_v;
9484         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9485                                      misc_parameters_3);
9486         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9487         if (inner) {
9488                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9489                                          inner_headers);
9490                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9491         } else {
9492                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9493                                          outer_headers);
9494                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9495         }
9496         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9497         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
9498         if (!icmp_v)
9499                 return;
9500         if (!icmp_m)
9501                 icmp_m = &rte_flow_item_icmp_mask;
9502         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
9503                  icmp_m->hdr.icmp_type);
9504         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
9505                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
9506         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
9507                  icmp_m->hdr.icmp_code);
9508         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
9509                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
9510         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
9511         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
9512         if (icmp_header_data_m) {
9513                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
9514                 icmp_header_data_v |=
9515                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
9516                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
9517                          icmp_header_data_m);
9518                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
9519                          icmp_header_data_v & icmp_header_data_m);
9520         }
9521 }
9522
9523 /**
9524  * Add GTP item to matcher and to the value.
9525  *
9526  * @param[in, out] matcher
9527  *   Flow matcher.
9528  * @param[in, out] key
9529  *   Flow matcher value.
9530  * @param[in] item
9531  *   Flow pattern to translate.
9532  * @param[in] inner
9533  *   Item is inner pattern.
9534  */
9535 static void
9536 flow_dv_translate_item_gtp(void *matcher, void *key,
9537                            const struct rte_flow_item *item, int inner)
9538 {
9539         const struct rte_flow_item_gtp *gtp_m = item->mask;
9540         const struct rte_flow_item_gtp *gtp_v = item->spec;
9541         void *headers_m;
9542         void *headers_v;
9543         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9544                                      misc_parameters_3);
9545         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9546         uint16_t dport = RTE_GTPU_UDP_PORT;
9547
9548         if (inner) {
9549                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9550                                          inner_headers);
9551                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9552         } else {
9553                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9554                                          outer_headers);
9555                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9556         }
9557         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9558                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9559                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9560         }
9561         if (!gtp_v)
9562                 return;
9563         if (!gtp_m)
9564                 gtp_m = &rte_flow_item_gtp_mask;
9565         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
9566                  gtp_m->v_pt_rsv_flags);
9567         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
9568                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
9569         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
9570         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
9571                  gtp_v->msg_type & gtp_m->msg_type);
9572         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
9573                  rte_be_to_cpu_32(gtp_m->teid));
9574         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
9575                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
9576 }
9577
9578 /**
9579  * Add GTP PSC item to matcher.
9580  *
9581  * @param[in, out] matcher
9582  *   Flow matcher.
9583  * @param[in, out] key
9584  *   Flow matcher value.
9585  * @param[in] item
9586  *   Flow pattern to translate.
9587  */
9588 static int
9589 flow_dv_translate_item_gtp_psc(void *matcher, void *key,
9590                                const struct rte_flow_item *item)
9591 {
9592         const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask;
9593         const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec;
9594         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9595                         misc_parameters_3);
9596         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9597         union {
9598                 uint32_t w32;
9599                 struct {
9600                         uint16_t seq_num;
9601                         uint8_t npdu_num;
9602                         uint8_t next_ext_header_type;
9603                 };
9604         } dw_2;
9605         uint8_t gtp_flags;
9606
9607         /* Always set E-flag match on one, regardless of GTP item settings. */
9608         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags);
9609         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9610         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags);
9611         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags);
9612         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9613         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags);
9614         /*Set next extension header type. */
9615         dw_2.seq_num = 0;
9616         dw_2.npdu_num = 0;
9617         dw_2.next_ext_header_type = 0xff;
9618         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2,
9619                  rte_cpu_to_be_32(dw_2.w32));
9620         dw_2.seq_num = 0;
9621         dw_2.npdu_num = 0;
9622         dw_2.next_ext_header_type = 0x85;
9623         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2,
9624                  rte_cpu_to_be_32(dw_2.w32));
9625         if (gtp_psc_v) {
9626                 union {
9627                         uint32_t w32;
9628                         struct {
9629                                 uint8_t len;
9630                                 uint8_t type_flags;
9631                                 uint8_t qfi;
9632                                 uint8_t reserved;
9633                         };
9634                 } dw_0;
9635
9636                 /*Set extension header PDU type and Qos. */
9637                 if (!gtp_psc_m)
9638                         gtp_psc_m = &rte_flow_item_gtp_psc_mask;
9639                 dw_0.w32 = 0;
9640                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->pdu_type);
9641                 dw_0.qfi = gtp_psc_m->qfi;
9642                 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0,
9643                          rte_cpu_to_be_32(dw_0.w32));
9644                 dw_0.w32 = 0;
9645                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->pdu_type &
9646                                                         gtp_psc_m->pdu_type);
9647                 dw_0.qfi = gtp_psc_v->qfi & gtp_psc_m->qfi;
9648                 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0,
9649                          rte_cpu_to_be_32(dw_0.w32));
9650         }
9651         return 0;
9652 }
9653
9654 /**
9655  * Add eCPRI item to matcher and to the value.
9656  *
9657  * @param[in] dev
9658  *   The devich to configure through.
9659  * @param[in, out] matcher
9660  *   Flow matcher.
9661  * @param[in, out] key
9662  *   Flow matcher value.
9663  * @param[in] item
9664  *   Flow pattern to translate.
9665  * @param[in] samples
9666  *   Sample IDs to be used in the matching.
9667  */
9668 static void
9669 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
9670                              void *key, const struct rte_flow_item *item)
9671 {
9672         struct mlx5_priv *priv = dev->data->dev_private;
9673         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
9674         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
9675         struct rte_ecpri_common_hdr common;
9676         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
9677                                      misc_parameters_4);
9678         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
9679         uint32_t *samples;
9680         void *dw_m;
9681         void *dw_v;
9682
9683         if (!ecpri_v)
9684                 return;
9685         if (!ecpri_m)
9686                 ecpri_m = &rte_flow_item_ecpri_mask;
9687         /*
9688          * Maximal four DW samples are supported in a single matching now.
9689          * Two are used now for a eCPRI matching:
9690          * 1. Type: one byte, mask should be 0x00ff0000 in network order
9691          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
9692          *    if any.
9693          */
9694         if (!ecpri_m->hdr.common.u32)
9695                 return;
9696         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
9697         /* Need to take the whole DW as the mask to fill the entry. */
9698         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9699                             prog_sample_field_value_0);
9700         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9701                             prog_sample_field_value_0);
9702         /* Already big endian (network order) in the header. */
9703         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
9704         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32;
9705         /* Sample#0, used for matching type, offset 0. */
9706         MLX5_SET(fte_match_set_misc4, misc4_m,
9707                  prog_sample_field_id_0, samples[0]);
9708         /* It makes no sense to set the sample ID in the mask field. */
9709         MLX5_SET(fte_match_set_misc4, misc4_v,
9710                  prog_sample_field_id_0, samples[0]);
9711         /*
9712          * Checking if message body part needs to be matched.
9713          * Some wildcard rules only matching type field should be supported.
9714          */
9715         if (ecpri_m->hdr.dummy[0]) {
9716                 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32);
9717                 switch (common.type) {
9718                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
9719                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
9720                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
9721                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9722                                             prog_sample_field_value_1);
9723                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9724                                             prog_sample_field_value_1);
9725                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
9726                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] &
9727                                             ecpri_m->hdr.dummy[0];
9728                         /* Sample#1, to match message body, offset 4. */
9729                         MLX5_SET(fte_match_set_misc4, misc4_m,
9730                                  prog_sample_field_id_1, samples[1]);
9731                         MLX5_SET(fte_match_set_misc4, misc4_v,
9732                                  prog_sample_field_id_1, samples[1]);
9733                         break;
9734                 default:
9735                         /* Others, do not match any sample ID. */
9736                         break;
9737                 }
9738         }
9739 }
9740
9741 /*
9742  * Add connection tracking status item to matcher
9743  *
9744  * @param[in] dev
9745  *   The devich to configure through.
9746  * @param[in, out] matcher
9747  *   Flow matcher.
9748  * @param[in, out] key
9749  *   Flow matcher value.
9750  * @param[in] item
9751  *   Flow pattern to translate.
9752  */
9753 static void
9754 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev,
9755                               void *matcher, void *key,
9756                               const struct rte_flow_item *item)
9757 {
9758         uint32_t reg_value = 0;
9759         int reg_id;
9760         /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */
9761         uint32_t reg_mask = 0;
9762         const struct rte_flow_item_conntrack *spec = item->spec;
9763         const struct rte_flow_item_conntrack *mask = item->mask;
9764         uint32_t flags;
9765         struct rte_flow_error error;
9766
9767         if (!mask)
9768                 mask = &rte_flow_item_conntrack_mask;
9769         if (!spec || !mask->flags)
9770                 return;
9771         flags = spec->flags & mask->flags;
9772         /* The conflict should be checked in the validation. */
9773         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID)
9774                 reg_value |= MLX5_CT_SYNDROME_VALID;
9775         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
9776                 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE;
9777         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID)
9778                 reg_value |= MLX5_CT_SYNDROME_INVALID;
9779         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)
9780                 reg_value |= MLX5_CT_SYNDROME_TRAP;
9781         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
9782                 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET;
9783         if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID |
9784                            RTE_FLOW_CONNTRACK_PKT_STATE_INVALID |
9785                            RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))
9786                 reg_mask |= 0xc0;
9787         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
9788                 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE;
9789         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
9790                 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET;
9791         /* The REG_C_x value could be saved during startup. */
9792         reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error);
9793         if (reg_id == REG_NON)
9794                 return;
9795         flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id,
9796                                reg_value, reg_mask);
9797 }
9798
9799 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
9800
9801 #define HEADER_IS_ZERO(match_criteria, headers)                              \
9802         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
9803                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
9804
9805 /**
9806  * Calculate flow matcher enable bitmap.
9807  *
9808  * @param match_criteria
9809  *   Pointer to flow matcher criteria.
9810  *
9811  * @return
9812  *   Bitmap of enabled fields.
9813  */
9814 static uint8_t
9815 flow_dv_matcher_enable(uint32_t *match_criteria)
9816 {
9817         uint8_t match_criteria_enable;
9818
9819         match_criteria_enable =
9820                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
9821                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
9822         match_criteria_enable |=
9823                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
9824                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
9825         match_criteria_enable |=
9826                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
9827                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
9828         match_criteria_enable |=
9829                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
9830                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9831         match_criteria_enable |=
9832                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
9833                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
9834         match_criteria_enable |=
9835                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
9836                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
9837         return match_criteria_enable;
9838 }
9839
9840 struct mlx5_hlist_entry *
9841 flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
9842 {
9843         struct mlx5_dev_ctx_shared *sh = list->ctx;
9844         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
9845         struct rte_eth_dev *dev = ctx->dev;
9846         struct mlx5_flow_tbl_data_entry *tbl_data;
9847         struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data;
9848         struct rte_flow_error *error = ctx->error;
9849         union mlx5_flow_tbl_key key = { .v64 = key64 };
9850         struct mlx5_flow_tbl_resource *tbl;
9851         void *domain;
9852         uint32_t idx = 0;
9853         int ret;
9854
9855         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
9856         if (!tbl_data) {
9857                 rte_flow_error_set(error, ENOMEM,
9858                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9859                                    NULL,
9860                                    "cannot allocate flow table data entry");
9861                 return NULL;
9862         }
9863         tbl_data->idx = idx;
9864         tbl_data->tunnel = tt_prm->tunnel;
9865         tbl_data->group_id = tt_prm->group_id;
9866         tbl_data->external = !!tt_prm->external;
9867         tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
9868         tbl_data->is_egress = !!key.is_egress;
9869         tbl_data->is_transfer = !!key.is_fdb;
9870         tbl_data->dummy = !!key.dummy;
9871         tbl_data->level = key.level;
9872         tbl_data->id = key.id;
9873         tbl = &tbl_data->tbl;
9874         if (key.dummy)
9875                 return &tbl_data->entry;
9876         if (key.is_fdb)
9877                 domain = sh->fdb_domain;
9878         else if (key.is_egress)
9879                 domain = sh->tx_domain;
9880         else
9881                 domain = sh->rx_domain;
9882         ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
9883         if (ret) {
9884                 rte_flow_error_set(error, ENOMEM,
9885                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9886                                    NULL, "cannot create flow table object");
9887                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
9888                 return NULL;
9889         }
9890         if (key.level != 0) {
9891                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
9892                                         (tbl->obj, &tbl_data->jump.action);
9893                 if (ret) {
9894                         rte_flow_error_set(error, ENOMEM,
9895                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9896                                            NULL,
9897                                            "cannot create flow jump action");
9898                         mlx5_flow_os_destroy_flow_tbl(tbl->obj);
9899                         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
9900                         return NULL;
9901                 }
9902         }
9903         MKSTR(matcher_name, "%s_%s_%u_%u_matcher_cache",
9904               key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
9905               key.level, key.id);
9906         mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh,
9907                              flow_dv_matcher_create_cb,
9908                              flow_dv_matcher_match_cb,
9909                              flow_dv_matcher_remove_cb);
9910         return &tbl_data->entry;
9911 }
9912
9913 int
9914 flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
9915                      struct mlx5_hlist_entry *entry, uint64_t key64,
9916                      void *cb_ctx __rte_unused)
9917 {
9918         struct mlx5_flow_tbl_data_entry *tbl_data =
9919                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
9920         union mlx5_flow_tbl_key key = { .v64 = key64 };
9921
9922         return tbl_data->level != key.level ||
9923                tbl_data->id != key.id ||
9924                tbl_data->dummy != key.dummy ||
9925                tbl_data->is_transfer != !!key.is_fdb ||
9926                tbl_data->is_egress != !!key.is_egress;
9927 }
9928
9929 /**
9930  * Get a flow table.
9931  *
9932  * @param[in, out] dev
9933  *   Pointer to rte_eth_dev structure.
9934  * @param[in] table_level
9935  *   Table level to use.
9936  * @param[in] egress
9937  *   Direction of the table.
9938  * @param[in] transfer
9939  *   E-Switch or NIC flow.
9940  * @param[in] dummy
9941  *   Dummy entry for dv API.
9942  * @param[in] table_id
9943  *   Table id to use.
9944  * @param[out] error
9945  *   pointer to error structure.
9946  *
9947  * @return
9948  *   Returns tables resource based on the index, NULL in case of failed.
9949  */
9950 struct mlx5_flow_tbl_resource *
9951 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
9952                          uint32_t table_level, uint8_t egress,
9953                          uint8_t transfer,
9954                          bool external,
9955                          const struct mlx5_flow_tunnel *tunnel,
9956                          uint32_t group_id, uint8_t dummy,
9957                          uint32_t table_id,
9958                          struct rte_flow_error *error)
9959 {
9960         struct mlx5_priv *priv = dev->data->dev_private;
9961         union mlx5_flow_tbl_key table_key = {
9962                 {
9963                         .level = table_level,
9964                         .id = table_id,
9965                         .reserved = 0,
9966                         .dummy = !!dummy,
9967                         .is_fdb = !!transfer,
9968                         .is_egress = !!egress,
9969                 }
9970         };
9971         struct mlx5_flow_tbl_tunnel_prm tt_prm = {
9972                 .tunnel = tunnel,
9973                 .group_id = group_id,
9974                 .external = external,
9975         };
9976         struct mlx5_flow_cb_ctx ctx = {
9977                 .dev = dev,
9978                 .error = error,
9979                 .data = &tt_prm,
9980         };
9981         struct mlx5_hlist_entry *entry;
9982         struct mlx5_flow_tbl_data_entry *tbl_data;
9983
9984         entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
9985         if (!entry) {
9986                 rte_flow_error_set(error, ENOMEM,
9987                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9988                                    "cannot get table");
9989                 return NULL;
9990         }
9991         DRV_LOG(DEBUG, "table_level %u table_id %u "
9992                 "tunnel %u group %u registered.",
9993                 table_level, table_id,
9994                 tunnel ? tunnel->tunnel_id : 0, group_id);
9995         tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
9996         return &tbl_data->tbl;
9997 }
9998
9999 void
10000 flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
10001                       struct mlx5_hlist_entry *entry)
10002 {
10003         struct mlx5_dev_ctx_shared *sh = list->ctx;
10004         struct mlx5_flow_tbl_data_entry *tbl_data =
10005                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10006
10007         MLX5_ASSERT(entry && sh);
10008         if (tbl_data->jump.action)
10009                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10010         if (tbl_data->tbl.obj)
10011                 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
10012         if (tbl_data->tunnel_offload && tbl_data->external) {
10013                 struct mlx5_hlist_entry *he;
10014                 struct mlx5_hlist *tunnel_grp_hash;
10015                 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10016                 union tunnel_tbl_key tunnel_key = {
10017                         .tunnel_id = tbl_data->tunnel ?
10018                                         tbl_data->tunnel->tunnel_id : 0,
10019                         .group = tbl_data->group_id
10020                 };
10021                 uint32_t table_level = tbl_data->level;
10022
10023                 tunnel_grp_hash = tbl_data->tunnel ?
10024                                         tbl_data->tunnel->groups :
10025                                         thub->groups;
10026                 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
10027                 if (he)
10028                         mlx5_hlist_unregister(tunnel_grp_hash, he);
10029                 DRV_LOG(DEBUG,
10030                         "table_level %u id %u tunnel %u group %u released.",
10031                         table_level,
10032                         tbl_data->id,
10033                         tbl_data->tunnel ?
10034                         tbl_data->tunnel->tunnel_id : 0,
10035                         tbl_data->group_id);
10036         }
10037         mlx5_cache_list_destroy(&tbl_data->matchers);
10038         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10039 }
10040
10041 /**
10042  * Release a flow table.
10043  *
10044  * @param[in] sh
10045  *   Pointer to device shared structure.
10046  * @param[in] tbl
10047  *   Table resource to be released.
10048  *
10049  * @return
10050  *   Returns 0 if table was released, else return 1;
10051  */
10052 static int
10053 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
10054                              struct mlx5_flow_tbl_resource *tbl)
10055 {
10056         struct mlx5_flow_tbl_data_entry *tbl_data =
10057                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10058
10059         if (!tbl)
10060                 return 0;
10061         return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
10062 }
10063
10064 int
10065 flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
10066                          struct mlx5_cache_entry *entry, void *cb_ctx)
10067 {
10068         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10069         struct mlx5_flow_dv_matcher *ref = ctx->data;
10070         struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur),
10071                                                         entry);
10072
10073         return cur->crc != ref->crc ||
10074                cur->priority != ref->priority ||
10075                memcmp((const void *)cur->mask.buf,
10076                       (const void *)ref->mask.buf, ref->mask.size);
10077 }
10078
10079 struct mlx5_cache_entry *
10080 flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
10081                           struct mlx5_cache_entry *entry __rte_unused,
10082                           void *cb_ctx)
10083 {
10084         struct mlx5_dev_ctx_shared *sh = list->ctx;
10085         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10086         struct mlx5_flow_dv_matcher *ref = ctx->data;
10087         struct mlx5_flow_dv_matcher *cache;
10088         struct mlx5dv_flow_matcher_attr dv_attr = {
10089                 .type = IBV_FLOW_ATTR_NORMAL,
10090                 .match_mask = (void *)&ref->mask,
10091         };
10092         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10093                                                             typeof(*tbl), tbl);
10094         int ret;
10095
10096         cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY);
10097         if (!cache) {
10098                 rte_flow_error_set(ctx->error, ENOMEM,
10099                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10100                                    "cannot create matcher");
10101                 return NULL;
10102         }
10103         *cache = *ref;
10104         dv_attr.match_criteria_enable =
10105                 flow_dv_matcher_enable(cache->mask.buf);
10106         dv_attr.priority = ref->priority;
10107         if (tbl->is_egress)
10108                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
10109         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
10110                                                &cache->matcher_object);
10111         if (ret) {
10112                 mlx5_free(cache);
10113                 rte_flow_error_set(ctx->error, ENOMEM,
10114                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10115                                    "cannot create matcher");
10116                 return NULL;
10117         }
10118         return &cache->entry;
10119 }
10120
10121 /**
10122  * Register the flow matcher.
10123  *
10124  * @param[in, out] dev
10125  *   Pointer to rte_eth_dev structure.
10126  * @param[in, out] matcher
10127  *   Pointer to flow matcher.
10128  * @param[in, out] key
10129  *   Pointer to flow table key.
10130  * @parm[in, out] dev_flow
10131  *   Pointer to the dev_flow.
10132  * @param[out] error
10133  *   pointer to error structure.
10134  *
10135  * @return
10136  *   0 on success otherwise -errno and errno is set.
10137  */
10138 static int
10139 flow_dv_matcher_register(struct rte_eth_dev *dev,
10140                          struct mlx5_flow_dv_matcher *ref,
10141                          union mlx5_flow_tbl_key *key,
10142                          struct mlx5_flow *dev_flow,
10143                          const struct mlx5_flow_tunnel *tunnel,
10144                          uint32_t group_id,
10145                          struct rte_flow_error *error)
10146 {
10147         struct mlx5_cache_entry *entry;
10148         struct mlx5_flow_dv_matcher *cache;
10149         struct mlx5_flow_tbl_resource *tbl;
10150         struct mlx5_flow_tbl_data_entry *tbl_data;
10151         struct mlx5_flow_cb_ctx ctx = {
10152                 .error = error,
10153                 .data = ref,
10154         };
10155
10156         /**
10157          * tunnel offload API requires this registration for cases when
10158          * tunnel match rule was inserted before tunnel set rule.
10159          */
10160         tbl = flow_dv_tbl_resource_get(dev, key->level,
10161                                        key->is_egress, key->is_fdb,
10162                                        dev_flow->external, tunnel,
10163                                        group_id, 0, key->id, error);
10164         if (!tbl)
10165                 return -rte_errno;      /* No need to refill the error info */
10166         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10167         ref->tbl = tbl;
10168         entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
10169         if (!entry) {
10170                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
10171                 return rte_flow_error_set(error, ENOMEM,
10172                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10173                                           "cannot allocate ref memory");
10174         }
10175         cache = container_of(entry, typeof(*cache), entry);
10176         dev_flow->handle->dvh.matcher = cache;
10177         return 0;
10178 }
10179
10180 struct mlx5_hlist_entry *
10181 flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
10182 {
10183         struct mlx5_dev_ctx_shared *sh = list->ctx;
10184         struct rte_flow_error *error = ctx;
10185         struct mlx5_flow_dv_tag_resource *entry;
10186         uint32_t idx = 0;
10187         int ret;
10188
10189         entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10190         if (!entry) {
10191                 rte_flow_error_set(error, ENOMEM,
10192                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10193                                    "cannot allocate resource memory");
10194                 return NULL;
10195         }
10196         entry->idx = idx;
10197         entry->tag_id = key;
10198         ret = mlx5_flow_os_create_flow_action_tag(key,
10199                                                   &entry->action);
10200         if (ret) {
10201                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
10202                 rte_flow_error_set(error, ENOMEM,
10203                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10204                                    NULL, "cannot create action");
10205                 return NULL;
10206         }
10207         return &entry->entry;
10208 }
10209
10210 int
10211 flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused,
10212                      struct mlx5_hlist_entry *entry, uint64_t key,
10213                      void *cb_ctx __rte_unused)
10214 {
10215         struct mlx5_flow_dv_tag_resource *tag =
10216                 container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10217
10218         return key != tag->tag_id;
10219 }
10220
10221 /**
10222  * Find existing tag resource or create and register a new one.
10223  *
10224  * @param dev[in, out]
10225  *   Pointer to rte_eth_dev structure.
10226  * @param[in, out] tag_be24
10227  *   Tag value in big endian then R-shift 8.
10228  * @parm[in, out] dev_flow
10229  *   Pointer to the dev_flow.
10230  * @param[out] error
10231  *   pointer to error structure.
10232  *
10233  * @return
10234  *   0 on success otherwise -errno and errno is set.
10235  */
10236 static int
10237 flow_dv_tag_resource_register
10238                         (struct rte_eth_dev *dev,
10239                          uint32_t tag_be24,
10240                          struct mlx5_flow *dev_flow,
10241                          struct rte_flow_error *error)
10242 {
10243         struct mlx5_priv *priv = dev->data->dev_private;
10244         struct mlx5_flow_dv_tag_resource *cache_resource;
10245         struct mlx5_hlist_entry *entry;
10246
10247         entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
10248         if (entry) {
10249                 cache_resource = container_of
10250                         (entry, struct mlx5_flow_dv_tag_resource, entry);
10251                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
10252                 dev_flow->dv.tag_resource = cache_resource;
10253                 return 0;
10254         }
10255         return -rte_errno;
10256 }
10257
10258 void
10259 flow_dv_tag_remove_cb(struct mlx5_hlist *list,
10260                       struct mlx5_hlist_entry *entry)
10261 {
10262         struct mlx5_dev_ctx_shared *sh = list->ctx;
10263         struct mlx5_flow_dv_tag_resource *tag =
10264                 container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10265
10266         MLX5_ASSERT(tag && sh && tag->action);
10267         claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
10268         DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag);
10269         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10270 }
10271
10272 /**
10273  * Release the tag.
10274  *
10275  * @param dev
10276  *   Pointer to Ethernet device.
10277  * @param tag_idx
10278  *   Tag index.
10279  *
10280  * @return
10281  *   1 while a reference on it exists, 0 when freed.
10282  */
10283 static int
10284 flow_dv_tag_release(struct rte_eth_dev *dev,
10285                     uint32_t tag_idx)
10286 {
10287         struct mlx5_priv *priv = dev->data->dev_private;
10288         struct mlx5_flow_dv_tag_resource *tag;
10289
10290         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
10291         if (!tag)
10292                 return 0;
10293         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
10294                 dev->data->port_id, (void *)tag, tag->entry.ref_cnt);
10295         return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry);
10296 }
10297
10298 /**
10299  * Translate port ID action to vport.
10300  *
10301  * @param[in] dev
10302  *   Pointer to rte_eth_dev structure.
10303  * @param[in] action
10304  *   Pointer to the port ID action.
10305  * @param[out] dst_port_id
10306  *   The target port ID.
10307  * @param[out] error
10308  *   Pointer to the error structure.
10309  *
10310  * @return
10311  *   0 on success, a negative errno value otherwise and rte_errno is set.
10312  */
10313 static int
10314 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
10315                                  const struct rte_flow_action *action,
10316                                  uint32_t *dst_port_id,
10317                                  struct rte_flow_error *error)
10318 {
10319         uint32_t port;
10320         struct mlx5_priv *priv;
10321         const struct rte_flow_action_port_id *conf =
10322                         (const struct rte_flow_action_port_id *)action->conf;
10323
10324         port = conf->original ? dev->data->port_id : conf->id;
10325         priv = mlx5_port_to_eswitch_info(port, false);
10326         if (!priv)
10327                 return rte_flow_error_set(error, -rte_errno,
10328                                           RTE_FLOW_ERROR_TYPE_ACTION,
10329                                           NULL,
10330                                           "No eswitch info was found for port");
10331 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
10332         /*
10333          * This parameter is transferred to
10334          * mlx5dv_dr_action_create_dest_ib_port().
10335          */
10336         *dst_port_id = priv->dev_port;
10337 #else
10338         /*
10339          * Legacy mode, no LAG configurations is supported.
10340          * This parameter is transferred to
10341          * mlx5dv_dr_action_create_dest_vport().
10342          */
10343         *dst_port_id = priv->vport_id;
10344 #endif
10345         return 0;
10346 }
10347
10348 /**
10349  * Create a counter with aging configuration.
10350  *
10351  * @param[in] dev
10352  *   Pointer to rte_eth_dev structure.
10353  * @param[in] dev_flow
10354  *   Pointer to the mlx5_flow.
10355  * @param[out] count
10356  *   Pointer to the counter action configuration.
10357  * @param[in] age
10358  *   Pointer to the aging action configuration.
10359  *
10360  * @return
10361  *   Index to flow counter on success, 0 otherwise.
10362  */
10363 static uint32_t
10364 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
10365                                 struct mlx5_flow *dev_flow,
10366                                 const struct rte_flow_action_count *count,
10367                                 const struct rte_flow_action_age *age)
10368 {
10369         uint32_t counter;
10370         struct mlx5_age_param *age_param;
10371
10372         if (count && count->shared)
10373                 counter = flow_dv_counter_get_shared(dev, count->id);
10374         else
10375                 counter = flow_dv_counter_alloc(dev, !!age);
10376         if (!counter || age == NULL)
10377                 return counter;
10378         age_param = flow_dv_counter_idx_get_age(dev, counter);
10379         age_param->context = age->context ? age->context :
10380                 (void *)(uintptr_t)(dev_flow->flow_idx);
10381         age_param->timeout = age->timeout;
10382         age_param->port_id = dev->data->port_id;
10383         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
10384         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
10385         return counter;
10386 }
10387
10388 /**
10389  * Add Tx queue matcher
10390  *
10391  * @param[in] dev
10392  *   Pointer to the dev struct.
10393  * @param[in, out] matcher
10394  *   Flow matcher.
10395  * @param[in, out] key
10396  *   Flow matcher value.
10397  * @param[in] item
10398  *   Flow pattern to translate.
10399  * @param[in] inner
10400  *   Item is inner pattern.
10401  */
10402 static void
10403 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
10404                                 void *matcher, void *key,
10405                                 const struct rte_flow_item *item)
10406 {
10407         const struct mlx5_rte_flow_item_tx_queue *queue_m;
10408         const struct mlx5_rte_flow_item_tx_queue *queue_v;
10409         void *misc_m =
10410                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
10411         void *misc_v =
10412                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
10413         struct mlx5_txq_ctrl *txq;
10414         uint32_t queue;
10415
10416
10417         queue_m = (const void *)item->mask;
10418         if (!queue_m)
10419                 return;
10420         queue_v = (const void *)item->spec;
10421         if (!queue_v)
10422                 return;
10423         txq = mlx5_txq_get(dev, queue_v->queue);
10424         if (!txq)
10425                 return;
10426         queue = txq->obj->sq->id;
10427         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
10428         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
10429                  queue & queue_m->queue);
10430         mlx5_txq_release(dev, queue_v->queue);
10431 }
10432
10433 /**
10434  * Set the hash fields according to the @p flow information.
10435  *
10436  * @param[in] dev_flow
10437  *   Pointer to the mlx5_flow.
10438  * @param[in] rss_desc
10439  *   Pointer to the mlx5_flow_rss_desc.
10440  */
10441 static void
10442 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
10443                        struct mlx5_flow_rss_desc *rss_desc)
10444 {
10445         uint64_t items = dev_flow->handle->layers;
10446         int rss_inner = 0;
10447         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
10448
10449         dev_flow->hash_fields = 0;
10450 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
10451         if (rss_desc->level >= 2) {
10452                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
10453                 rss_inner = 1;
10454         }
10455 #endif
10456         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
10457             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
10458                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
10459                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10460                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
10461                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10462                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
10463                         else
10464                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
10465                 }
10466         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
10467                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
10468                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
10469                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10470                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
10471                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10472                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
10473                         else
10474                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
10475                 }
10476         }
10477         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
10478             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
10479                 if (rss_types & ETH_RSS_UDP) {
10480                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10481                                 dev_flow->hash_fields |=
10482                                                 IBV_RX_HASH_SRC_PORT_UDP;
10483                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10484                                 dev_flow->hash_fields |=
10485                                                 IBV_RX_HASH_DST_PORT_UDP;
10486                         else
10487                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
10488                 }
10489         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
10490                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
10491                 if (rss_types & ETH_RSS_TCP) {
10492                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10493                                 dev_flow->hash_fields |=
10494                                                 IBV_RX_HASH_SRC_PORT_TCP;
10495                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10496                                 dev_flow->hash_fields |=
10497                                                 IBV_RX_HASH_DST_PORT_TCP;
10498                         else
10499                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
10500                 }
10501         }
10502 }
10503
10504 /**
10505  * Prepare an Rx Hash queue.
10506  *
10507  * @param dev
10508  *   Pointer to Ethernet device.
10509  * @param[in] dev_flow
10510  *   Pointer to the mlx5_flow.
10511  * @param[in] rss_desc
10512  *   Pointer to the mlx5_flow_rss_desc.
10513  * @param[out] hrxq_idx
10514  *   Hash Rx queue index.
10515  *
10516  * @return
10517  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
10518  */
10519 static struct mlx5_hrxq *
10520 flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
10521                      struct mlx5_flow *dev_flow,
10522                      struct mlx5_flow_rss_desc *rss_desc,
10523                      uint32_t *hrxq_idx)
10524 {
10525         struct mlx5_priv *priv = dev->data->dev_private;
10526         struct mlx5_flow_handle *dh = dev_flow->handle;
10527         struct mlx5_hrxq *hrxq;
10528
10529         MLX5_ASSERT(rss_desc->queue_num);
10530         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
10531         rss_desc->hash_fields = dev_flow->hash_fields;
10532         rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL);
10533         rss_desc->shared_rss = 0;
10534         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
10535         if (!*hrxq_idx)
10536                 return NULL;
10537         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10538                               *hrxq_idx);
10539         return hrxq;
10540 }
10541
10542 /**
10543  * Release sample sub action resource.
10544  *
10545  * @param[in, out] dev
10546  *   Pointer to rte_eth_dev structure.
10547  * @param[in] act_res
10548  *   Pointer to sample sub action resource.
10549  */
10550 static void
10551 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
10552                                    struct mlx5_flow_sub_actions_idx *act_res)
10553 {
10554         if (act_res->rix_hrxq) {
10555                 mlx5_hrxq_release(dev, act_res->rix_hrxq);
10556                 act_res->rix_hrxq = 0;
10557         }
10558         if (act_res->rix_encap_decap) {
10559                 flow_dv_encap_decap_resource_release(dev,
10560                                                      act_res->rix_encap_decap);
10561                 act_res->rix_encap_decap = 0;
10562         }
10563         if (act_res->rix_port_id_action) {
10564                 flow_dv_port_id_action_resource_release(dev,
10565                                                 act_res->rix_port_id_action);
10566                 act_res->rix_port_id_action = 0;
10567         }
10568         if (act_res->rix_tag) {
10569                 flow_dv_tag_release(dev, act_res->rix_tag);
10570                 act_res->rix_tag = 0;
10571         }
10572         if (act_res->rix_jump) {
10573                 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump);
10574                 act_res->rix_jump = 0;
10575         }
10576 }
10577
10578 int
10579 flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused,
10580                         struct mlx5_cache_entry *entry, void *cb_ctx)
10581 {
10582         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10583         struct rte_eth_dev *dev = ctx->dev;
10584         struct mlx5_flow_dv_sample_resource *resource = ctx->data;
10585         struct mlx5_flow_dv_sample_resource *cache_resource =
10586                         container_of(entry, typeof(*cache_resource), entry);
10587
10588         if (resource->ratio == cache_resource->ratio &&
10589             resource->ft_type == cache_resource->ft_type &&
10590             resource->ft_id == cache_resource->ft_id &&
10591             resource->set_action == cache_resource->set_action &&
10592             !memcmp((void *)&resource->sample_act,
10593                     (void *)&cache_resource->sample_act,
10594                     sizeof(struct mlx5_flow_sub_actions_list))) {
10595                 /*
10596                  * Existing sample action should release the prepared
10597                  * sub-actions reference counter.
10598                  */
10599                 flow_dv_sample_sub_actions_release(dev,
10600                                                 &resource->sample_idx);
10601                 return 0;
10602         }
10603         return 1;
10604 }
10605
10606 struct mlx5_cache_entry *
10607 flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
10608                          struct mlx5_cache_entry *entry __rte_unused,
10609                          void *cb_ctx)
10610 {
10611         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10612         struct rte_eth_dev *dev = ctx->dev;
10613         struct mlx5_flow_dv_sample_resource *resource = ctx->data;
10614         void **sample_dv_actions = resource->sub_actions;
10615         struct mlx5_flow_dv_sample_resource *cache_resource;
10616         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
10617         struct mlx5_priv *priv = dev->data->dev_private;
10618         struct mlx5_dev_ctx_shared *sh = priv->sh;
10619         struct mlx5_flow_tbl_resource *tbl;
10620         uint32_t idx = 0;
10621         const uint32_t next_ft_step = 1;
10622         uint32_t next_ft_id = resource->ft_id + next_ft_step;
10623         uint8_t is_egress = 0;
10624         uint8_t is_transfer = 0;
10625         struct rte_flow_error *error = ctx->error;
10626
10627         /* Register new sample resource. */
10628         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
10629         if (!cache_resource) {
10630                 rte_flow_error_set(error, ENOMEM,
10631                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10632                                           NULL,
10633                                           "cannot allocate resource memory");
10634                 return NULL;
10635         }
10636         *cache_resource = *resource;
10637         /* Create normal path table level */
10638         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
10639                 is_transfer = 1;
10640         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
10641                 is_egress = 1;
10642         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
10643                                         is_egress, is_transfer,
10644                                         true, NULL, 0, 0, 0, error);
10645         if (!tbl) {
10646                 rte_flow_error_set(error, ENOMEM,
10647                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10648                                           NULL,
10649                                           "fail to create normal path table "
10650                                           "for sample");
10651                 goto error;
10652         }
10653         cache_resource->normal_path_tbl = tbl;
10654         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10655                 if (!sh->default_miss_action) {
10656                         rte_flow_error_set(error, ENOMEM,
10657                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10658                                                 NULL,
10659                                                 "default miss action was not "
10660                                                 "created");
10661                         goto error;
10662                 }
10663                 sample_dv_actions[resource->sample_act.actions_num++] =
10664                                                 sh->default_miss_action;
10665         }
10666         /* Create a DR sample action */
10667         sampler_attr.sample_ratio = cache_resource->ratio;
10668         sampler_attr.default_next_table = tbl->obj;
10669         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
10670         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
10671                                                         &sample_dv_actions[0];
10672         sampler_attr.action = cache_resource->set_action;
10673         if (mlx5_os_flow_dr_create_flow_action_sampler
10674                         (&sampler_attr, &cache_resource->verbs_action)) {
10675                 rte_flow_error_set(error, ENOMEM,
10676                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10677                                         NULL, "cannot create sample action");
10678                 goto error;
10679         }
10680         cache_resource->idx = idx;
10681         cache_resource->dev = dev;
10682         return &cache_resource->entry;
10683 error:
10684         if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
10685                 flow_dv_sample_sub_actions_release(dev,
10686                                                    &cache_resource->sample_idx);
10687         if (cache_resource->normal_path_tbl)
10688                 flow_dv_tbl_resource_release(MLX5_SH(dev),
10689                                 cache_resource->normal_path_tbl);
10690         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10691         return NULL;
10692
10693 }
10694
10695 /**
10696  * Find existing sample resource or create and register a new one.
10697  *
10698  * @param[in, out] dev
10699  *   Pointer to rte_eth_dev structure.
10700  * @param[in] resource
10701  *   Pointer to sample resource.
10702  * @parm[in, out] dev_flow
10703  *   Pointer to the dev_flow.
10704  * @param[out] error
10705  *   pointer to error structure.
10706  *
10707  * @return
10708  *   0 on success otherwise -errno and errno is set.
10709  */
10710 static int
10711 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
10712                          struct mlx5_flow_dv_sample_resource *resource,
10713                          struct mlx5_flow *dev_flow,
10714                          struct rte_flow_error *error)
10715 {
10716         struct mlx5_flow_dv_sample_resource *cache_resource;
10717         struct mlx5_cache_entry *entry;
10718         struct mlx5_priv *priv = dev->data->dev_private;
10719         struct mlx5_flow_cb_ctx ctx = {
10720                 .dev = dev,
10721                 .error = error,
10722                 .data = resource,
10723         };
10724
10725         entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx);
10726         if (!entry)
10727                 return -rte_errno;
10728         cache_resource = container_of(entry, typeof(*cache_resource), entry);
10729         dev_flow->handle->dvh.rix_sample = cache_resource->idx;
10730         dev_flow->dv.sample_res = cache_resource;
10731         return 0;
10732 }
10733
10734 int
10735 flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused,
10736                             struct mlx5_cache_entry *entry, void *cb_ctx)
10737 {
10738         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10739         struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
10740         struct rte_eth_dev *dev = ctx->dev;
10741         struct mlx5_flow_dv_dest_array_resource *cache_resource =
10742                         container_of(entry, typeof(*cache_resource), entry);
10743         uint32_t idx = 0;
10744
10745         if (resource->num_of_dest == cache_resource->num_of_dest &&
10746             resource->ft_type == cache_resource->ft_type &&
10747             !memcmp((void *)cache_resource->sample_act,
10748                     (void *)resource->sample_act,
10749                    (resource->num_of_dest *
10750                    sizeof(struct mlx5_flow_sub_actions_list)))) {
10751                 /*
10752                  * Existing sample action should release the prepared
10753                  * sub-actions reference counter.
10754                  */
10755                 for (idx = 0; idx < resource->num_of_dest; idx++)
10756                         flow_dv_sample_sub_actions_release(dev,
10757                                         &resource->sample_idx[idx]);
10758                 return 0;
10759         }
10760         return 1;
10761 }
10762
10763 struct mlx5_cache_entry *
10764 flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
10765                          struct mlx5_cache_entry *entry __rte_unused,
10766                          void *cb_ctx)
10767 {
10768         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10769         struct rte_eth_dev *dev = ctx->dev;
10770         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10771         struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
10772         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
10773         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
10774         struct mlx5_priv *priv = dev->data->dev_private;
10775         struct mlx5_dev_ctx_shared *sh = priv->sh;
10776         struct mlx5_flow_sub_actions_list *sample_act;
10777         struct mlx5dv_dr_domain *domain;
10778         uint32_t idx = 0, res_idx = 0;
10779         struct rte_flow_error *error = ctx->error;
10780         uint64_t action_flags;
10781         int ret;
10782
10783         /* Register new destination array resource. */
10784         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10785                                             &res_idx);
10786         if (!cache_resource) {
10787                 rte_flow_error_set(error, ENOMEM,
10788                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10789                                           NULL,
10790                                           "cannot allocate resource memory");
10791                 return NULL;
10792         }
10793         *cache_resource = *resource;
10794         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
10795                 domain = sh->fdb_domain;
10796         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
10797                 domain = sh->rx_domain;
10798         else
10799                 domain = sh->tx_domain;
10800         for (idx = 0; idx < resource->num_of_dest; idx++) {
10801                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
10802                                  mlx5_malloc(MLX5_MEM_ZERO,
10803                                  sizeof(struct mlx5dv_dr_action_dest_attr),
10804                                  0, SOCKET_ID_ANY);
10805                 if (!dest_attr[idx]) {
10806                         rte_flow_error_set(error, ENOMEM,
10807                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10808                                            NULL,
10809                                            "cannot allocate resource memory");
10810                         goto error;
10811                 }
10812                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
10813                 sample_act = &resource->sample_act[idx];
10814                 action_flags = sample_act->action_flags;
10815                 switch (action_flags) {
10816                 case MLX5_FLOW_ACTION_QUEUE:
10817                         dest_attr[idx]->dest = sample_act->dr_queue_action;
10818                         break;
10819                 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP):
10820                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
10821                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
10822                         dest_attr[idx]->dest_reformat->reformat =
10823                                         sample_act->dr_encap_action;
10824                         dest_attr[idx]->dest_reformat->dest =
10825                                         sample_act->dr_port_id_action;
10826                         break;
10827                 case MLX5_FLOW_ACTION_PORT_ID:
10828                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
10829                         break;
10830                 case MLX5_FLOW_ACTION_JUMP:
10831                         dest_attr[idx]->dest = sample_act->dr_jump_action;
10832                         break;
10833                 default:
10834                         rte_flow_error_set(error, EINVAL,
10835                                            RTE_FLOW_ERROR_TYPE_ACTION,
10836                                            NULL,
10837                                            "unsupported actions type");
10838                         goto error;
10839                 }
10840         }
10841         /* create a dest array actioin */
10842         ret = mlx5_os_flow_dr_create_flow_action_dest_array
10843                                                 (domain,
10844                                                  cache_resource->num_of_dest,
10845                                                  dest_attr,
10846                                                  &cache_resource->action);
10847         if (ret) {
10848                 rte_flow_error_set(error, ENOMEM,
10849                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10850                                    NULL,
10851                                    "cannot create destination array action");
10852                 goto error;
10853         }
10854         cache_resource->idx = res_idx;
10855         cache_resource->dev = dev;
10856         for (idx = 0; idx < resource->num_of_dest; idx++)
10857                 mlx5_free(dest_attr[idx]);
10858         return &cache_resource->entry;
10859 error:
10860         for (idx = 0; idx < resource->num_of_dest; idx++) {
10861                 flow_dv_sample_sub_actions_release(dev,
10862                                 &cache_resource->sample_idx[idx]);
10863                 if (dest_attr[idx])
10864                         mlx5_free(dest_attr[idx]);
10865         }
10866
10867         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
10868         return NULL;
10869 }
10870
10871 /**
10872  * Find existing destination array resource or create and register a new one.
10873  *
10874  * @param[in, out] dev
10875  *   Pointer to rte_eth_dev structure.
10876  * @param[in] resource
10877  *   Pointer to destination array resource.
10878  * @parm[in, out] dev_flow
10879  *   Pointer to the dev_flow.
10880  * @param[out] error
10881  *   pointer to error structure.
10882  *
10883  * @return
10884  *   0 on success otherwise -errno and errno is set.
10885  */
10886 static int
10887 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
10888                          struct mlx5_flow_dv_dest_array_resource *resource,
10889                          struct mlx5_flow *dev_flow,
10890                          struct rte_flow_error *error)
10891 {
10892         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10893         struct mlx5_priv *priv = dev->data->dev_private;
10894         struct mlx5_cache_entry *entry;
10895         struct mlx5_flow_cb_ctx ctx = {
10896                 .dev = dev,
10897                 .error = error,
10898                 .data = resource,
10899         };
10900
10901         entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx);
10902         if (!entry)
10903                 return -rte_errno;
10904         cache_resource = container_of(entry, typeof(*cache_resource), entry);
10905         dev_flow->handle->dvh.rix_dest_array = cache_resource->idx;
10906         dev_flow->dv.dest_array_res = cache_resource;
10907         return 0;
10908 }
10909
10910 /**
10911  * Convert Sample action to DV specification.
10912  *
10913  * @param[in] dev
10914  *   Pointer to rte_eth_dev structure.
10915  * @param[in] action
10916  *   Pointer to sample action structure.
10917  * @param[in, out] dev_flow
10918  *   Pointer to the mlx5_flow.
10919  * @param[in] attr
10920  *   Pointer to the flow attributes.
10921  * @param[in, out] num_of_dest
10922  *   Pointer to the num of destination.
10923  * @param[in, out] sample_actions
10924  *   Pointer to sample actions list.
10925  * @param[in, out] res
10926  *   Pointer to sample resource.
10927  * @param[out] error
10928  *   Pointer to the error structure.
10929  *
10930  * @return
10931  *   0 on success, a negative errno value otherwise and rte_errno is set.
10932  */
10933 static int
10934 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
10935                                 const struct rte_flow_action_sample *action,
10936                                 struct mlx5_flow *dev_flow,
10937                                 const struct rte_flow_attr *attr,
10938                                 uint32_t *num_of_dest,
10939                                 void **sample_actions,
10940                                 struct mlx5_flow_dv_sample_resource *res,
10941                                 struct rte_flow_error *error)
10942 {
10943         struct mlx5_priv *priv = dev->data->dev_private;
10944         const struct rte_flow_action *sub_actions;
10945         struct mlx5_flow_sub_actions_list *sample_act;
10946         struct mlx5_flow_sub_actions_idx *sample_idx;
10947         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
10948         struct rte_flow *flow = dev_flow->flow;
10949         struct mlx5_flow_rss_desc *rss_desc;
10950         uint64_t action_flags = 0;
10951
10952         MLX5_ASSERT(wks);
10953         rss_desc = &wks->rss_desc;
10954         sample_act = &res->sample_act;
10955         sample_idx = &res->sample_idx;
10956         res->ratio = action->ratio;
10957         sub_actions = action->actions;
10958         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
10959                 int type = sub_actions->type;
10960                 uint32_t pre_rix = 0;
10961                 void *pre_r;
10962                 switch (type) {
10963                 case RTE_FLOW_ACTION_TYPE_QUEUE:
10964                 {
10965                         const struct rte_flow_action_queue *queue;
10966                         struct mlx5_hrxq *hrxq;
10967                         uint32_t hrxq_idx;
10968
10969                         queue = sub_actions->conf;
10970                         rss_desc->queue_num = 1;
10971                         rss_desc->queue[0] = queue->index;
10972                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
10973                                                     rss_desc, &hrxq_idx);
10974                         if (!hrxq)
10975                                 return rte_flow_error_set
10976                                         (error, rte_errno,
10977                                          RTE_FLOW_ERROR_TYPE_ACTION,
10978                                          NULL,
10979                                          "cannot create fate queue");
10980                         sample_act->dr_queue_action = hrxq->action;
10981                         sample_idx->rix_hrxq = hrxq_idx;
10982                         sample_actions[sample_act->actions_num++] =
10983                                                 hrxq->action;
10984                         (*num_of_dest)++;
10985                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
10986                         if (action_flags & MLX5_FLOW_ACTION_MARK)
10987                                 dev_flow->handle->rix_hrxq = hrxq_idx;
10988                         dev_flow->handle->fate_action =
10989                                         MLX5_FLOW_FATE_QUEUE;
10990                         break;
10991                 }
10992                 case RTE_FLOW_ACTION_TYPE_RSS:
10993                 {
10994                         struct mlx5_hrxq *hrxq;
10995                         uint32_t hrxq_idx;
10996                         const struct rte_flow_action_rss *rss;
10997                         const uint8_t *rss_key;
10998
10999                         rss = sub_actions->conf;
11000                         memcpy(rss_desc->queue, rss->queue,
11001                                rss->queue_num * sizeof(uint16_t));
11002                         rss_desc->queue_num = rss->queue_num;
11003                         /* NULL RSS key indicates default RSS key. */
11004                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11005                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11006                         /*
11007                          * rss->level and rss.types should be set in advance
11008                          * when expanding items for RSS.
11009                          */
11010                         flow_dv_hashfields_set(dev_flow, rss_desc);
11011                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11012                                                     rss_desc, &hrxq_idx);
11013                         if (!hrxq)
11014                                 return rte_flow_error_set
11015                                         (error, rte_errno,
11016                                          RTE_FLOW_ERROR_TYPE_ACTION,
11017                                          NULL,
11018                                          "cannot create fate queue");
11019                         sample_act->dr_queue_action = hrxq->action;
11020                         sample_idx->rix_hrxq = hrxq_idx;
11021                         sample_actions[sample_act->actions_num++] =
11022                                                 hrxq->action;
11023                         (*num_of_dest)++;
11024                         action_flags |= MLX5_FLOW_ACTION_RSS;
11025                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11026                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11027                         dev_flow->handle->fate_action =
11028                                         MLX5_FLOW_FATE_QUEUE;
11029                         break;
11030                 }
11031                 case RTE_FLOW_ACTION_TYPE_MARK:
11032                 {
11033                         uint32_t tag_be = mlx5_flow_mark_set
11034                                 (((const struct rte_flow_action_mark *)
11035                                 (sub_actions->conf))->id);
11036
11037                         dev_flow->handle->mark = 1;
11038                         pre_rix = dev_flow->handle->dvh.rix_tag;
11039                         /* Save the mark resource before sample */
11040                         pre_r = dev_flow->dv.tag_resource;
11041                         if (flow_dv_tag_resource_register(dev, tag_be,
11042                                                   dev_flow, error))
11043                                 return -rte_errno;
11044                         MLX5_ASSERT(dev_flow->dv.tag_resource);
11045                         sample_act->dr_tag_action =
11046                                 dev_flow->dv.tag_resource->action;
11047                         sample_idx->rix_tag =
11048                                 dev_flow->handle->dvh.rix_tag;
11049                         sample_actions[sample_act->actions_num++] =
11050                                                 sample_act->dr_tag_action;
11051                         /* Recover the mark resource after sample */
11052                         dev_flow->dv.tag_resource = pre_r;
11053                         dev_flow->handle->dvh.rix_tag = pre_rix;
11054                         action_flags |= MLX5_FLOW_ACTION_MARK;
11055                         break;
11056                 }
11057                 case RTE_FLOW_ACTION_TYPE_COUNT:
11058                 {
11059                         if (!flow->counter) {
11060                                 flow->counter =
11061                                         flow_dv_translate_create_counter(dev,
11062                                                 dev_flow, sub_actions->conf,
11063                                                 0);
11064                                 if (!flow->counter)
11065                                         return rte_flow_error_set
11066                                                 (error, rte_errno,
11067                                                 RTE_FLOW_ERROR_TYPE_ACTION,
11068                                                 NULL,
11069                                                 "cannot create counter"
11070                                                 " object.");
11071                         }
11072                         sample_act->dr_cnt_action =
11073                                   (flow_dv_counter_get_by_idx(dev,
11074                                   flow->counter, NULL))->action;
11075                         sample_actions[sample_act->actions_num++] =
11076                                                 sample_act->dr_cnt_action;
11077                         action_flags |= MLX5_FLOW_ACTION_COUNT;
11078                         break;
11079                 }
11080                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
11081                 {
11082                         struct mlx5_flow_dv_port_id_action_resource
11083                                         port_id_resource;
11084                         uint32_t port_id = 0;
11085
11086                         memset(&port_id_resource, 0, sizeof(port_id_resource));
11087                         /* Save the port id resource before sample */
11088                         pre_rix = dev_flow->handle->rix_port_id_action;
11089                         pre_r = dev_flow->dv.port_id_action;
11090                         if (flow_dv_translate_action_port_id(dev, sub_actions,
11091                                                              &port_id, error))
11092                                 return -rte_errno;
11093                         port_id_resource.port_id = port_id;
11094                         if (flow_dv_port_id_action_resource_register
11095                             (dev, &port_id_resource, dev_flow, error))
11096                                 return -rte_errno;
11097                         sample_act->dr_port_id_action =
11098                                 dev_flow->dv.port_id_action->action;
11099                         sample_idx->rix_port_id_action =
11100                                 dev_flow->handle->rix_port_id_action;
11101                         sample_actions[sample_act->actions_num++] =
11102                                                 sample_act->dr_port_id_action;
11103                         /* Recover the port id resource after sample */
11104                         dev_flow->dv.port_id_action = pre_r;
11105                         dev_flow->handle->rix_port_id_action = pre_rix;
11106                         (*num_of_dest)++;
11107                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
11108                         break;
11109                 }
11110                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
11111                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
11112                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
11113                         /* Save the encap resource before sample */
11114                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
11115                         pre_r = dev_flow->dv.encap_decap;
11116                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
11117                                                            dev_flow,
11118                                                            attr->transfer,
11119                                                            error))
11120                                 return -rte_errno;
11121                         sample_act->dr_encap_action =
11122                                 dev_flow->dv.encap_decap->action;
11123                         sample_idx->rix_encap_decap =
11124                                 dev_flow->handle->dvh.rix_encap_decap;
11125                         sample_actions[sample_act->actions_num++] =
11126                                                 sample_act->dr_encap_action;
11127                         /* Recover the encap resource after sample */
11128                         dev_flow->dv.encap_decap = pre_r;
11129                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
11130                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
11131                         break;
11132                 default:
11133                         return rte_flow_error_set(error, EINVAL,
11134                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11135                                 NULL,
11136                                 "Not support for sampler action");
11137                 }
11138         }
11139         sample_act->action_flags = action_flags;
11140         res->ft_id = dev_flow->dv.group;
11141         if (attr->transfer) {
11142                 union {
11143                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
11144                         uint64_t set_action;
11145                 } action_ctx = { .set_action = 0 };
11146
11147                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
11148                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
11149                          MLX5_MODIFICATION_TYPE_SET);
11150                 MLX5_SET(set_action_in, action_ctx.action_in, field,
11151                          MLX5_MODI_META_REG_C_0);
11152                 MLX5_SET(set_action_in, action_ctx.action_in, data,
11153                          priv->vport_meta_tag);
11154                 res->set_action = action_ctx.set_action;
11155         } else if (attr->ingress) {
11156                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
11157         } else {
11158                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
11159         }
11160         return 0;
11161 }
11162
11163 /**
11164  * Convert Sample action to DV specification.
11165  *
11166  * @param[in] dev
11167  *   Pointer to rte_eth_dev structure.
11168  * @param[in, out] dev_flow
11169  *   Pointer to the mlx5_flow.
11170  * @param[in] num_of_dest
11171  *   The num of destination.
11172  * @param[in, out] res
11173  *   Pointer to sample resource.
11174  * @param[in, out] mdest_res
11175  *   Pointer to destination array resource.
11176  * @param[in] sample_actions
11177  *   Pointer to sample path actions list.
11178  * @param[in] action_flags
11179  *   Holds the actions detected until now.
11180  * @param[out] error
11181  *   Pointer to the error structure.
11182  *
11183  * @return
11184  *   0 on success, a negative errno value otherwise and rte_errno is set.
11185  */
11186 static int
11187 flow_dv_create_action_sample(struct rte_eth_dev *dev,
11188                              struct mlx5_flow *dev_flow,
11189                              uint32_t num_of_dest,
11190                              struct mlx5_flow_dv_sample_resource *res,
11191                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
11192                              void **sample_actions,
11193                              uint64_t action_flags,
11194                              struct rte_flow_error *error)
11195 {
11196         /* update normal path action resource into last index of array */
11197         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
11198         struct mlx5_flow_sub_actions_list *sample_act =
11199                                         &mdest_res->sample_act[dest_index];
11200         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11201         struct mlx5_flow_rss_desc *rss_desc;
11202         uint32_t normal_idx = 0;
11203         struct mlx5_hrxq *hrxq;
11204         uint32_t hrxq_idx;
11205
11206         MLX5_ASSERT(wks);
11207         rss_desc = &wks->rss_desc;
11208         if (num_of_dest > 1) {
11209                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
11210                         /* Handle QP action for mirroring */
11211                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11212                                                     rss_desc, &hrxq_idx);
11213                         if (!hrxq)
11214                                 return rte_flow_error_set
11215                                      (error, rte_errno,
11216                                       RTE_FLOW_ERROR_TYPE_ACTION,
11217                                       NULL,
11218                                       "cannot create rx queue");
11219                         normal_idx++;
11220                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
11221                         sample_act->dr_queue_action = hrxq->action;
11222                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11223                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11224                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
11225                 }
11226                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
11227                         normal_idx++;
11228                         mdest_res->sample_idx[dest_index].rix_encap_decap =
11229                                 dev_flow->handle->dvh.rix_encap_decap;
11230                         sample_act->dr_encap_action =
11231                                 dev_flow->dv.encap_decap->action;
11232                         dev_flow->handle->dvh.rix_encap_decap = 0;
11233                 }
11234                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
11235                         normal_idx++;
11236                         mdest_res->sample_idx[dest_index].rix_port_id_action =
11237                                 dev_flow->handle->rix_port_id_action;
11238                         sample_act->dr_port_id_action =
11239                                 dev_flow->dv.port_id_action->action;
11240                         dev_flow->handle->rix_port_id_action = 0;
11241                 }
11242                 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) {
11243                         normal_idx++;
11244                         mdest_res->sample_idx[dest_index].rix_jump =
11245                                 dev_flow->handle->rix_jump;
11246                         sample_act->dr_jump_action =
11247                                 dev_flow->dv.jump->action;
11248                         dev_flow->handle->rix_jump = 0;
11249                 }
11250                 sample_act->actions_num = normal_idx;
11251                 /* update sample action resource into first index of array */
11252                 mdest_res->ft_type = res->ft_type;
11253                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
11254                                 sizeof(struct mlx5_flow_sub_actions_idx));
11255                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
11256                                 sizeof(struct mlx5_flow_sub_actions_list));
11257                 mdest_res->num_of_dest = num_of_dest;
11258                 if (flow_dv_dest_array_resource_register(dev, mdest_res,
11259                                                          dev_flow, error))
11260                         return rte_flow_error_set(error, EINVAL,
11261                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11262                                                   NULL, "can't create sample "
11263                                                   "action");
11264         } else {
11265                 res->sub_actions = sample_actions;
11266                 if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
11267                         return rte_flow_error_set(error, EINVAL,
11268                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11269                                                   NULL,
11270                                                   "can't create sample action");
11271         }
11272         return 0;
11273 }
11274
11275 /**
11276  * Remove an ASO age action from age actions list.
11277  *
11278  * @param[in] dev
11279  *   Pointer to the Ethernet device structure.
11280  * @param[in] age
11281  *   Pointer to the aso age action handler.
11282  */
11283 static void
11284 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev,
11285                                 struct mlx5_aso_age_action *age)
11286 {
11287         struct mlx5_age_info *age_info;
11288         struct mlx5_age_param *age_param = &age->age_params;
11289         struct mlx5_priv *priv = dev->data->dev_private;
11290         uint16_t expected = AGE_CANDIDATE;
11291
11292         age_info = GET_PORT_AGE_INFO(priv);
11293         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
11294                                          AGE_FREE, false, __ATOMIC_RELAXED,
11295                                          __ATOMIC_RELAXED)) {
11296                 /**
11297                  * We need the lock even it is age timeout,
11298                  * since age action may still in process.
11299                  */
11300                 rte_spinlock_lock(&age_info->aged_sl);
11301                 LIST_REMOVE(age, next);
11302                 rte_spinlock_unlock(&age_info->aged_sl);
11303                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
11304         }
11305 }
11306
11307 /**
11308  * Release an ASO age action.
11309  *
11310  * @param[in] dev
11311  *   Pointer to the Ethernet device structure.
11312  * @param[in] age_idx
11313  *   Index of ASO age action to release.
11314  * @param[in] flow
11315  *   True if the release operation is during flow destroy operation.
11316  *   False if the release operation is during action destroy operation.
11317  *
11318  * @return
11319  *   0 when age action was removed, otherwise the number of references.
11320  */
11321 static int
11322 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
11323 {
11324         struct mlx5_priv *priv = dev->data->dev_private;
11325         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11326         struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx);
11327         uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED);
11328
11329         if (!ret) {
11330                 flow_dv_aso_age_remove_from_age(dev, age);
11331                 rte_spinlock_lock(&mng->free_sl);
11332                 LIST_INSERT_HEAD(&mng->free, age, next);
11333                 rte_spinlock_unlock(&mng->free_sl);
11334         }
11335         return ret;
11336 }
11337
11338 /**
11339  * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
11340  *
11341  * @param[in] dev
11342  *   Pointer to the Ethernet device structure.
11343  *
11344  * @return
11345  *   0 on success, otherwise negative errno value and rte_errno is set.
11346  */
11347 static int
11348 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
11349 {
11350         struct mlx5_priv *priv = dev->data->dev_private;
11351         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11352         void *old_pools = mng->pools;
11353         uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
11354         uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
11355         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
11356
11357         if (!pools) {
11358                 rte_errno = ENOMEM;
11359                 return -ENOMEM;
11360         }
11361         if (old_pools) {
11362                 memcpy(pools, old_pools,
11363                        mng->n * sizeof(struct mlx5_flow_counter_pool *));
11364                 mlx5_free(old_pools);
11365         } else {
11366                 /* First ASO flow hit allocation - starting ASO data-path. */
11367                 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh);
11368
11369                 if (ret) {
11370                         mlx5_free(pools);
11371                         return ret;
11372                 }
11373         }
11374         mng->n = resize;
11375         mng->pools = pools;
11376         return 0;
11377 }
11378
11379 /**
11380  * Create and initialize a new ASO aging pool.
11381  *
11382  * @param[in] dev
11383  *   Pointer to the Ethernet device structure.
11384  * @param[out] age_free
11385  *   Where to put the pointer of a new age action.
11386  *
11387  * @return
11388  *   The age actions pool pointer and @p age_free is set on success,
11389  *   NULL otherwise and rte_errno is set.
11390  */
11391 static struct mlx5_aso_age_pool *
11392 flow_dv_age_pool_create(struct rte_eth_dev *dev,
11393                         struct mlx5_aso_age_action **age_free)
11394 {
11395         struct mlx5_priv *priv = dev->data->dev_private;
11396         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11397         struct mlx5_aso_age_pool *pool = NULL;
11398         struct mlx5_devx_obj *obj = NULL;
11399         uint32_t i;
11400
11401         obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->ctx,
11402                                                     priv->sh->pdn);
11403         if (!obj) {
11404                 rte_errno = ENODATA;
11405                 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX.");
11406                 return NULL;
11407         }
11408         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
11409         if (!pool) {
11410                 claim_zero(mlx5_devx_cmd_destroy(obj));
11411                 rte_errno = ENOMEM;
11412                 return NULL;
11413         }
11414         pool->flow_hit_aso_obj = obj;
11415         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
11416         rte_spinlock_lock(&mng->resize_sl);
11417         pool->index = mng->next;
11418         /* Resize pools array if there is no room for the new pool in it. */
11419         if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) {
11420                 claim_zero(mlx5_devx_cmd_destroy(obj));
11421                 mlx5_free(pool);
11422                 rte_spinlock_unlock(&mng->resize_sl);
11423                 return NULL;
11424         }
11425         mng->pools[pool->index] = pool;
11426         mng->next++;
11427         rte_spinlock_unlock(&mng->resize_sl);
11428         /* Assign the first action in the new pool, the rest go to free list. */
11429         *age_free = &pool->actions[0];
11430         for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) {
11431                 pool->actions[i].offset = i;
11432                 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next);
11433         }
11434         return pool;
11435 }
11436
11437 /**
11438  * Allocate a ASO aging bit.
11439  *
11440  * @param[in] dev
11441  *   Pointer to the Ethernet device structure.
11442  * @param[out] error
11443  *   Pointer to the error structure.
11444  *
11445  * @return
11446  *   Index to ASO age action on success, 0 otherwise and rte_errno is set.
11447  */
11448 static uint32_t
11449 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
11450 {
11451         struct mlx5_priv *priv = dev->data->dev_private;
11452         const struct mlx5_aso_age_pool *pool;
11453         struct mlx5_aso_age_action *age_free = NULL;
11454         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11455
11456         MLX5_ASSERT(mng);
11457         /* Try to get the next free age action bit. */
11458         rte_spinlock_lock(&mng->free_sl);
11459         age_free = LIST_FIRST(&mng->free);
11460         if (age_free) {
11461                 LIST_REMOVE(age_free, next);
11462         } else if (!flow_dv_age_pool_create(dev, &age_free)) {
11463                 rte_spinlock_unlock(&mng->free_sl);
11464                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
11465                                    NULL, "failed to create ASO age pool");
11466                 return 0; /* 0 is an error. */
11467         }
11468         rte_spinlock_unlock(&mng->free_sl);
11469         pool = container_of
11470           ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL])
11471                   (age_free - age_free->offset), const struct mlx5_aso_age_pool,
11472                                                                        actions);
11473         if (!age_free->dr_action) {
11474                 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0,
11475                                                  error);
11476
11477                 if (reg_c < 0) {
11478                         rte_flow_error_set(error, rte_errno,
11479                                            RTE_FLOW_ERROR_TYPE_ACTION,
11480                                            NULL, "failed to get reg_c "
11481                                            "for ASO flow hit");
11482                         return 0; /* 0 is an error. */
11483                 }
11484 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
11485                 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso
11486                                 (priv->sh->rx_domain,
11487                                  pool->flow_hit_aso_obj->obj, age_free->offset,
11488                                  MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET,
11489                                  (reg_c - REG_C_0));
11490 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
11491                 if (!age_free->dr_action) {
11492                         rte_errno = errno;
11493                         rte_spinlock_lock(&mng->free_sl);
11494                         LIST_INSERT_HEAD(&mng->free, age_free, next);
11495                         rte_spinlock_unlock(&mng->free_sl);
11496                         rte_flow_error_set(error, rte_errno,
11497                                            RTE_FLOW_ERROR_TYPE_ACTION,
11498                                            NULL, "failed to create ASO "
11499                                            "flow hit action");
11500                         return 0; /* 0 is an error. */
11501                 }
11502         }
11503         __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED);
11504         return pool->index | ((age_free->offset + 1) << 16);
11505 }
11506
11507 /**
11508  * Initialize flow ASO age parameters.
11509  *
11510  * @param[in] dev
11511  *   Pointer to rte_eth_dev structure.
11512  * @param[in] age_idx
11513  *   Index of ASO age action.
11514  * @param[in] context
11515  *   Pointer to flow counter age context.
11516  * @param[in] timeout
11517  *   Aging timeout in seconds.
11518  *
11519  */
11520 static void
11521 flow_dv_aso_age_params_init(struct rte_eth_dev *dev,
11522                             uint32_t age_idx,
11523                             void *context,
11524                             uint32_t timeout)
11525 {
11526         struct mlx5_aso_age_action *aso_age;
11527
11528         aso_age = flow_aso_age_get_by_idx(dev, age_idx);
11529         MLX5_ASSERT(aso_age);
11530         aso_age->age_params.context = context;
11531         aso_age->age_params.timeout = timeout;
11532         aso_age->age_params.port_id = dev->data->port_id;
11533         __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0,
11534                          __ATOMIC_RELAXED);
11535         __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE,
11536                          __ATOMIC_RELAXED);
11537 }
11538
11539 static void
11540 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask,
11541                                const struct rte_flow_item_integrity *value,
11542                                void *headers_m, void *headers_v)
11543 {
11544         if (mask->l4_ok) {
11545                 /* application l4_ok filter aggregates all hardware l4 filters
11546                  * therefore hw l4_checksum_ok must be implicitly added here.
11547                  */
11548                 struct rte_flow_item_integrity local_item;
11549
11550                 local_item.l4_csum_ok = 1;
11551                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
11552                          local_item.l4_csum_ok);
11553                 if (value->l4_ok) {
11554                         /* application l4_ok = 1 matches sets both hw flags
11555                          * l4_ok and l4_checksum_ok flags to 1.
11556                          */
11557                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11558                                  l4_checksum_ok, local_item.l4_csum_ok);
11559                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok,
11560                                  mask->l4_ok);
11561                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok,
11562                                  value->l4_ok);
11563                 } else {
11564                         /* application l4_ok = 0 matches on hw flag
11565                          * l4_checksum_ok = 0 only.
11566                          */
11567                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11568                                  l4_checksum_ok, 0);
11569                 }
11570         } else if (mask->l4_csum_ok) {
11571                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
11572                          mask->l4_csum_ok);
11573                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
11574                          value->l4_csum_ok);
11575         }
11576 }
11577
11578 static void
11579 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask,
11580                                const struct rte_flow_item_integrity *value,
11581                                void *headers_m, void *headers_v,
11582                                bool is_ipv4)
11583 {
11584         if (mask->l3_ok) {
11585                 /* application l3_ok filter aggregates all hardware l3 filters
11586                  * therefore hw ipv4_checksum_ok must be implicitly added here.
11587                  */
11588                 struct rte_flow_item_integrity local_item;
11589
11590                 local_item.ipv4_csum_ok = !!is_ipv4;
11591                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
11592                          local_item.ipv4_csum_ok);
11593                 if (value->l3_ok) {
11594                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11595                                  ipv4_checksum_ok, local_item.ipv4_csum_ok);
11596                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok,
11597                                  mask->l3_ok);
11598                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok,
11599                                  value->l3_ok);
11600                 } else {
11601                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11602                                  ipv4_checksum_ok, 0);
11603                 }
11604         } else if (mask->ipv4_csum_ok) {
11605                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
11606                          mask->ipv4_csum_ok);
11607                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
11608                          value->ipv4_csum_ok);
11609         }
11610 }
11611
11612 static void
11613 flow_dv_translate_item_integrity(void *matcher, void *key,
11614                                  const struct rte_flow_item *head_item,
11615                                  const struct rte_flow_item *integrity_item)
11616 {
11617         const struct rte_flow_item_integrity *mask = integrity_item->mask;
11618         const struct rte_flow_item_integrity *value = integrity_item->spec;
11619         const struct rte_flow_item *tunnel_item, *end_item, *item;
11620         void *headers_m;
11621         void *headers_v;
11622         uint32_t l3_protocol;
11623
11624         if (!value)
11625                 return;
11626         if (!mask)
11627                 mask = &rte_flow_item_integrity_mask;
11628         if (value->level > 1) {
11629                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
11630                                          inner_headers);
11631                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
11632         } else {
11633                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
11634                                          outer_headers);
11635                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
11636         }
11637         tunnel_item = mlx5_flow_find_tunnel_item(head_item);
11638         if (value->level > 1) {
11639                 /* tunnel item was verified during the item validation */
11640                 item = tunnel_item;
11641                 end_item = mlx5_find_end_item(tunnel_item);
11642         } else {
11643                 item = head_item;
11644                 end_item = tunnel_item ? tunnel_item :
11645                            mlx5_find_end_item(integrity_item);
11646         }
11647         l3_protocol = mask->l3_ok ?
11648                       mlx5_flow_locate_proto_l3(&item, end_item) : 0;
11649         flow_dv_translate_integrity_l3(mask, value, headers_m, headers_v,
11650                                        l3_protocol == RTE_ETHER_TYPE_IPV4);
11651         flow_dv_translate_integrity_l4(mask, value, headers_m, headers_v);
11652 }
11653
11654 /**
11655  * Prepares DV flow counter with aging configuration.
11656  * Gets it by index when exists, creates a new one when doesn't.
11657  *
11658  * @param[in] dev
11659  *   Pointer to rte_eth_dev structure.
11660  * @param[in] dev_flow
11661  *   Pointer to the mlx5_flow.
11662  * @param[in, out] flow
11663  *   Pointer to the sub flow.
11664  * @param[in] count
11665  *   Pointer to the counter action configuration.
11666  * @param[in] age
11667  *   Pointer to the aging action configuration.
11668  * @param[out] error
11669  *   Pointer to the error structure.
11670  *
11671  * @return
11672  *   Pointer to the counter, NULL otherwise.
11673  */
11674 static struct mlx5_flow_counter *
11675 flow_dv_prepare_counter(struct rte_eth_dev *dev,
11676                         struct mlx5_flow *dev_flow,
11677                         struct rte_flow *flow,
11678                         const struct rte_flow_action_count *count,
11679                         const struct rte_flow_action_age *age,
11680                         struct rte_flow_error *error)
11681 {
11682         if (!flow->counter) {
11683                 flow->counter = flow_dv_translate_create_counter(dev, dev_flow,
11684                                                                  count, age);
11685                 if (!flow->counter) {
11686                         rte_flow_error_set(error, rte_errno,
11687                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11688                                            "cannot create counter object.");
11689                         return NULL;
11690                 }
11691         }
11692         return flow_dv_counter_get_by_idx(dev, flow->counter, NULL);
11693 }
11694
11695 /*
11696  * Release an ASO CT action by its own device.
11697  *
11698  * @param[in] dev
11699  *   Pointer to the Ethernet device structure.
11700  * @param[in] idx
11701  *   Index of ASO CT action to release.
11702  *
11703  * @return
11704  *   0 when CT action was removed, otherwise the number of references.
11705  */
11706 static inline int
11707 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx)
11708 {
11709         struct mlx5_priv *priv = dev->data->dev_private;
11710         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
11711         uint32_t ret;
11712         struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx);
11713         enum mlx5_aso_ct_state state =
11714                         __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
11715
11716         /* Cannot release when CT is in the ASO SQ. */
11717         if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY)
11718                 return -1;
11719         ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED);
11720         if (!ret) {
11721                 if (ct->dr_action_orig) {
11722 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
11723                         claim_zero(mlx5_glue->destroy_flow_action
11724                                         (ct->dr_action_orig));
11725 #endif
11726                         ct->dr_action_orig = NULL;
11727                 }
11728                 if (ct->dr_action_rply) {
11729 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
11730                         claim_zero(mlx5_glue->destroy_flow_action
11731                                         (ct->dr_action_rply));
11732 #endif
11733                         ct->dr_action_rply = NULL;
11734                 }
11735                 /* Clear the state to free, no need in 1st allocation. */
11736                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE);
11737                 rte_spinlock_lock(&mng->ct_sl);
11738                 LIST_INSERT_HEAD(&mng->free_cts, ct, next);
11739                 rte_spinlock_unlock(&mng->ct_sl);
11740         }
11741         return (int)ret;
11742 }
11743
11744 static inline int
11745 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx)
11746 {
11747         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx);
11748         uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx);
11749         struct rte_eth_dev *owndev = &rte_eth_devices[owner];
11750         RTE_SET_USED(dev);
11751
11752         MLX5_ASSERT(owner < RTE_MAX_ETHPORTS);
11753         if (dev->data->dev_started != 1)
11754                 return -1;
11755         return flow_dv_aso_ct_dev_release(owndev, idx);
11756 }
11757
11758 /*
11759  * Resize the ASO CT pools array by 64 pools.
11760  *
11761  * @param[in] dev
11762  *   Pointer to the Ethernet device structure.
11763  *
11764  * @return
11765  *   0 on success, otherwise negative errno value and rte_errno is set.
11766  */
11767 static int
11768 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev)
11769 {
11770         struct mlx5_priv *priv = dev->data->dev_private;
11771         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
11772         void *old_pools = mng->pools;
11773         /* Magic number now, need a macro. */
11774         uint32_t resize = mng->n + 64;
11775         uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize;
11776         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
11777
11778         if (!pools) {
11779                 rte_errno = ENOMEM;
11780                 return -rte_errno;
11781         }
11782         rte_rwlock_write_lock(&mng->resize_rwl);
11783         /* ASO SQ/QP was already initialized in the startup. */
11784         if (old_pools) {
11785                 /* Realloc could be an alternative choice. */
11786                 rte_memcpy(pools, old_pools,
11787                            mng->n * sizeof(struct mlx5_aso_ct_pool *));
11788                 mlx5_free(old_pools);
11789         }
11790         mng->n = resize;
11791         mng->pools = pools;
11792         rte_rwlock_write_unlock(&mng->resize_rwl);
11793         return 0;
11794 }
11795
11796 /*
11797  * Create and initialize a new ASO CT pool.
11798  *
11799  * @param[in] dev
11800  *   Pointer to the Ethernet device structure.
11801  * @param[out] ct_free
11802  *   Where to put the pointer of a new CT action.
11803  *
11804  * @return
11805  *   The CT actions pool pointer and @p ct_free is set on success,
11806  *   NULL otherwise and rte_errno is set.
11807  */
11808 static struct mlx5_aso_ct_pool *
11809 flow_dv_ct_pool_create(struct rte_eth_dev *dev,
11810                        struct mlx5_aso_ct_action **ct_free)
11811 {
11812         struct mlx5_priv *priv = dev->data->dev_private;
11813         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
11814         struct mlx5_aso_ct_pool *pool = NULL;
11815         struct mlx5_devx_obj *obj = NULL;
11816         uint32_t i;
11817         uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL);
11818
11819         obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->ctx,
11820                                                 priv->sh->pdn, log_obj_size);
11821         if (!obj) {
11822                 rte_errno = ENODATA;
11823                 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
11824                 return NULL;
11825         }
11826         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
11827         if (!pool) {
11828                 rte_errno = ENOMEM;
11829                 claim_zero(mlx5_devx_cmd_destroy(obj));
11830                 return NULL;
11831         }
11832         pool->devx_obj = obj;
11833         pool->index = mng->next;
11834         /* Resize pools array if there is no room for the new pool in it. */
11835         if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) {
11836                 claim_zero(mlx5_devx_cmd_destroy(obj));
11837                 mlx5_free(pool);
11838                 return NULL;
11839         }
11840         mng->pools[pool->index] = pool;
11841         mng->next++;
11842         /* Assign the first action in the new pool, the rest go to free list. */
11843         *ct_free = &pool->actions[0];
11844         /* Lock outside, the list operation is safe here. */
11845         for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) {
11846                 /* refcnt is 0 when allocating the memory. */
11847                 pool->actions[i].offset = i;
11848                 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next);
11849         }
11850         return pool;
11851 }
11852
11853 /*
11854  * Allocate a ASO CT action from free list.
11855  *
11856  * @param[in] dev
11857  *   Pointer to the Ethernet device structure.
11858  * @param[out] error
11859  *   Pointer to the error structure.
11860  *
11861  * @return
11862  *   Index to ASO CT action on success, 0 otherwise and rte_errno is set.
11863  */
11864 static uint32_t
11865 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
11866 {
11867         struct mlx5_priv *priv = dev->data->dev_private;
11868         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
11869         struct mlx5_aso_ct_action *ct = NULL;
11870         struct mlx5_aso_ct_pool *pool;
11871         uint8_t reg_c;
11872         uint32_t ct_idx;
11873
11874         MLX5_ASSERT(mng);
11875         if (!priv->config.devx) {
11876                 rte_errno = ENOTSUP;
11877                 return 0;
11878         }
11879         /* Get a free CT action, if no, a new pool will be created. */
11880         rte_spinlock_lock(&mng->ct_sl);
11881         ct = LIST_FIRST(&mng->free_cts);
11882         if (ct) {
11883                 LIST_REMOVE(ct, next);
11884         } else if (!flow_dv_ct_pool_create(dev, &ct)) {
11885                 rte_spinlock_unlock(&mng->ct_sl);
11886                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
11887                                    NULL, "failed to create ASO CT pool");
11888                 return 0;
11889         }
11890         rte_spinlock_unlock(&mng->ct_sl);
11891         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
11892         ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset);
11893         /* 0: inactive, 1: created, 2+: used by flows. */
11894         __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED);
11895         reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error);
11896         if (!ct->dr_action_orig) {
11897 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
11898                 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso
11899                         (priv->sh->rx_domain, pool->devx_obj->obj,
11900                          ct->offset,
11901                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR,
11902                          reg_c - REG_C_0);
11903 #else
11904                 RTE_SET_USED(reg_c);
11905 #endif
11906                 if (!ct->dr_action_orig) {
11907                         flow_dv_aso_ct_dev_release(dev, ct_idx);
11908                         rte_flow_error_set(error, rte_errno,
11909                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11910                                            "failed to create ASO CT action");
11911                         return 0;
11912                 }
11913         }
11914         if (!ct->dr_action_rply) {
11915 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
11916                 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso
11917                         (priv->sh->rx_domain, pool->devx_obj->obj,
11918                          ct->offset,
11919                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER,
11920                          reg_c - REG_C_0);
11921 #endif
11922                 if (!ct->dr_action_rply) {
11923                         flow_dv_aso_ct_dev_release(dev, ct_idx);
11924                         rte_flow_error_set(error, rte_errno,
11925                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11926                                            "failed to create ASO CT action");
11927                         return 0;
11928                 }
11929         }
11930         return ct_idx;
11931 }
11932
11933 /*
11934  * Create a conntrack object with context and actions by using ASO mechanism.
11935  *
11936  * @param[in] dev
11937  *   Pointer to rte_eth_dev structure.
11938  * @param[in] pro
11939  *   Pointer to conntrack information profile.
11940  * @param[out] error
11941  *   Pointer to the error structure.
11942  *
11943  * @return
11944  *   Index to conntrack object on success, 0 otherwise.
11945  */
11946 static uint32_t
11947 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev,
11948                                    const struct rte_flow_action_conntrack *pro,
11949                                    struct rte_flow_error *error)
11950 {
11951         struct mlx5_priv *priv = dev->data->dev_private;
11952         struct mlx5_dev_ctx_shared *sh = priv->sh;
11953         struct mlx5_aso_ct_action *ct;
11954         uint32_t idx;
11955
11956         if (!sh->ct_aso_en)
11957                 return rte_flow_error_set(error, ENOTSUP,
11958                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11959                                           "Connection is not supported");
11960         idx = flow_dv_aso_ct_alloc(dev, error);
11961         if (!idx)
11962                 return rte_flow_error_set(error, rte_errno,
11963                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11964                                           "Failed to allocate CT object");
11965         ct = flow_aso_ct_get_by_dev_idx(dev, idx);
11966         if (mlx5_aso_ct_update_by_wqe(sh, ct, pro))
11967                 return rte_flow_error_set(error, EBUSY,
11968                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11969                                           "Failed to update CT");
11970         ct->is_original = !!pro->is_original_dir;
11971         ct->peer = pro->peer_port;
11972         return idx;
11973 }
11974
11975 /**
11976  * Fill the flow with DV spec, lock free
11977  * (mutex should be acquired by caller).
11978  *
11979  * @param[in] dev
11980  *   Pointer to rte_eth_dev structure.
11981  * @param[in, out] dev_flow
11982  *   Pointer to the sub flow.
11983  * @param[in] attr
11984  *   Pointer to the flow attributes.
11985  * @param[in] items
11986  *   Pointer to the list of items.
11987  * @param[in] actions
11988  *   Pointer to the list of actions.
11989  * @param[out] error
11990  *   Pointer to the error structure.
11991  *
11992  * @return
11993  *   0 on success, a negative errno value otherwise and rte_errno is set.
11994  */
11995 static int
11996 flow_dv_translate(struct rte_eth_dev *dev,
11997                   struct mlx5_flow *dev_flow,
11998                   const struct rte_flow_attr *attr,
11999                   const struct rte_flow_item items[],
12000                   const struct rte_flow_action actions[],
12001                   struct rte_flow_error *error)
12002 {
12003         struct mlx5_priv *priv = dev->data->dev_private;
12004         struct mlx5_dev_config *dev_conf = &priv->config;
12005         struct rte_flow *flow = dev_flow->flow;
12006         struct mlx5_flow_handle *handle = dev_flow->handle;
12007         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12008         struct mlx5_flow_rss_desc *rss_desc;
12009         uint64_t item_flags = 0;
12010         uint64_t last_item = 0;
12011         uint64_t action_flags = 0;
12012         struct mlx5_flow_dv_matcher matcher = {
12013                 .mask = {
12014                         .size = sizeof(matcher.mask.buf) -
12015                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
12016                 },
12017         };
12018         int actions_n = 0;
12019         bool actions_end = false;
12020         union {
12021                 struct mlx5_flow_dv_modify_hdr_resource res;
12022                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
12023                             sizeof(struct mlx5_modification_cmd) *
12024                             (MLX5_MAX_MODIFY_NUM + 1)];
12025         } mhdr_dummy;
12026         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
12027         const struct rte_flow_action_count *count = NULL;
12028         const struct rte_flow_action_age *non_shared_age = NULL;
12029         union flow_dv_attr flow_attr = { .attr = 0 };
12030         uint32_t tag_be;
12031         union mlx5_flow_tbl_key tbl_key;
12032         uint32_t modify_action_position = UINT32_MAX;
12033         void *match_mask = matcher.mask.buf;
12034         void *match_value = dev_flow->dv.value.buf;
12035         uint8_t next_protocol = 0xff;
12036         struct rte_vlan_hdr vlan = { 0 };
12037         struct mlx5_flow_dv_dest_array_resource mdest_res;
12038         struct mlx5_flow_dv_sample_resource sample_res;
12039         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
12040         const struct rte_flow_action_sample *sample = NULL;
12041         struct mlx5_flow_sub_actions_list *sample_act;
12042         uint32_t sample_act_pos = UINT32_MAX;
12043         uint32_t age_act_pos = UINT32_MAX;
12044         uint32_t num_of_dest = 0;
12045         int tmp_actions_n = 0;
12046         uint32_t table;
12047         int ret = 0;
12048         const struct mlx5_flow_tunnel *tunnel = NULL;
12049         struct flow_grp_info grp_info = {
12050                 .external = !!dev_flow->external,
12051                 .transfer = !!attr->transfer,
12052                 .fdb_def_rule = !!priv->fdb_def_rule,
12053                 .skip_scale = dev_flow->skip_scale &
12054                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
12055                 .std_tbl_fix = true,
12056         };
12057         const struct rte_flow_item *head_item = items;
12058
12059         if (!wks)
12060                 return rte_flow_error_set(error, ENOMEM,
12061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12062                                           NULL,
12063                                           "failed to push flow workspace");
12064         rss_desc = &wks->rss_desc;
12065         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
12066         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
12067         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12068                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12069         /* update normal path action resource into last index of array */
12070         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
12071         if (is_tunnel_offload_active(dev)) {
12072                 if (dev_flow->tunnel) {
12073                         RTE_VERIFY(dev_flow->tof_type ==
12074                                    MLX5_TUNNEL_OFFLOAD_MISS_RULE);
12075                         tunnel = dev_flow->tunnel;
12076                 } else {
12077                         tunnel = mlx5_get_tof(items, actions,
12078                                               &dev_flow->tof_type);
12079                         dev_flow->tunnel = tunnel;
12080                 }
12081                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
12082                                         (dev, attr, tunnel, dev_flow->tof_type);
12083         }
12084         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12085                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12086         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
12087                                        &grp_info, error);
12088         if (ret)
12089                 return ret;
12090         dev_flow->dv.group = table;
12091         if (attr->transfer)
12092                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12093         /* number of actions must be set to 0 in case of dirty stack. */
12094         mhdr_res->actions_num = 0;
12095         if (is_flow_tunnel_match_rule(dev_flow->tof_type)) {
12096                 /*
12097                  * do not add decap action if match rule drops packet
12098                  * HW rejects rules with decap & drop
12099                  *
12100                  * if tunnel match rule was inserted before matching tunnel set
12101                  * rule flow table used in the match rule must be registered.
12102                  * current implementation handles that in the
12103                  * flow_dv_match_register() at the function end.
12104                  */
12105                 bool add_decap = true;
12106                 const struct rte_flow_action *ptr = actions;
12107
12108                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
12109                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
12110                                 add_decap = false;
12111                                 break;
12112                         }
12113                 }
12114                 if (add_decap) {
12115                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12116                                                            attr->transfer,
12117                                                            error))
12118                                 return -rte_errno;
12119                         dev_flow->dv.actions[actions_n++] =
12120                                         dev_flow->dv.encap_decap->action;
12121                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12122                 }
12123         }
12124         for (; !actions_end ; actions++) {
12125                 const struct rte_flow_action_queue *queue;
12126                 const struct rte_flow_action_rss *rss;
12127                 const struct rte_flow_action *action = actions;
12128                 const uint8_t *rss_key;
12129                 struct mlx5_flow_tbl_resource *tbl;
12130                 struct mlx5_aso_age_action *age_act;
12131                 struct mlx5_flow_counter *cnt_act;
12132                 uint32_t port_id = 0;
12133                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
12134                 int action_type = actions->type;
12135                 const struct rte_flow_action *found_action = NULL;
12136                 uint32_t jump_group = 0;
12137                 uint32_t owner_idx;
12138                 struct mlx5_aso_ct_action *ct;
12139
12140                 if (!mlx5_flow_os_action_supported(action_type))
12141                         return rte_flow_error_set(error, ENOTSUP,
12142                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12143                                                   actions,
12144                                                   "action not supported");
12145                 switch (action_type) {
12146                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
12147                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
12148                         break;
12149                 case RTE_FLOW_ACTION_TYPE_VOID:
12150                         break;
12151                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
12152                         if (flow_dv_translate_action_port_id(dev, action,
12153                                                              &port_id, error))
12154                                 return -rte_errno;
12155                         port_id_resource.port_id = port_id;
12156                         MLX5_ASSERT(!handle->rix_port_id_action);
12157                         if (flow_dv_port_id_action_resource_register
12158                             (dev, &port_id_resource, dev_flow, error))
12159                                 return -rte_errno;
12160                         dev_flow->dv.actions[actions_n++] =
12161                                         dev_flow->dv.port_id_action->action;
12162                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12163                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
12164                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12165                         num_of_dest++;
12166                         break;
12167                 case RTE_FLOW_ACTION_TYPE_FLAG:
12168                         action_flags |= MLX5_FLOW_ACTION_FLAG;
12169                         dev_flow->handle->mark = 1;
12170                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12171                                 struct rte_flow_action_mark mark = {
12172                                         .id = MLX5_FLOW_MARK_DEFAULT,
12173                                 };
12174
12175                                 if (flow_dv_convert_action_mark(dev, &mark,
12176                                                                 mhdr_res,
12177                                                                 error))
12178                                         return -rte_errno;
12179                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12180                                 break;
12181                         }
12182                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
12183                         /*
12184                          * Only one FLAG or MARK is supported per device flow
12185                          * right now. So the pointer to the tag resource must be
12186                          * zero before the register process.
12187                          */
12188                         MLX5_ASSERT(!handle->dvh.rix_tag);
12189                         if (flow_dv_tag_resource_register(dev, tag_be,
12190                                                           dev_flow, error))
12191                                 return -rte_errno;
12192                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12193                         dev_flow->dv.actions[actions_n++] =
12194                                         dev_flow->dv.tag_resource->action;
12195                         break;
12196                 case RTE_FLOW_ACTION_TYPE_MARK:
12197                         action_flags |= MLX5_FLOW_ACTION_MARK;
12198                         dev_flow->handle->mark = 1;
12199                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12200                                 const struct rte_flow_action_mark *mark =
12201                                         (const struct rte_flow_action_mark *)
12202                                                 actions->conf;
12203
12204                                 if (flow_dv_convert_action_mark(dev, mark,
12205                                                                 mhdr_res,
12206                                                                 error))
12207                                         return -rte_errno;
12208                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12209                                 break;
12210                         }
12211                         /* Fall-through */
12212                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
12213                         /* Legacy (non-extensive) MARK action. */
12214                         tag_be = mlx5_flow_mark_set
12215                               (((const struct rte_flow_action_mark *)
12216                                (actions->conf))->id);
12217                         MLX5_ASSERT(!handle->dvh.rix_tag);
12218                         if (flow_dv_tag_resource_register(dev, tag_be,
12219                                                           dev_flow, error))
12220                                 return -rte_errno;
12221                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12222                         dev_flow->dv.actions[actions_n++] =
12223                                         dev_flow->dv.tag_resource->action;
12224                         break;
12225                 case RTE_FLOW_ACTION_TYPE_SET_META:
12226                         if (flow_dv_convert_action_set_meta
12227                                 (dev, mhdr_res, attr,
12228                                  (const struct rte_flow_action_set_meta *)
12229                                   actions->conf, error))
12230                                 return -rte_errno;
12231                         action_flags |= MLX5_FLOW_ACTION_SET_META;
12232                         break;
12233                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
12234                         if (flow_dv_convert_action_set_tag
12235                                 (dev, mhdr_res,
12236                                  (const struct rte_flow_action_set_tag *)
12237                                   actions->conf, error))
12238                                 return -rte_errno;
12239                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12240                         break;
12241                 case RTE_FLOW_ACTION_TYPE_DROP:
12242                         action_flags |= MLX5_FLOW_ACTION_DROP;
12243                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
12244                         break;
12245                 case RTE_FLOW_ACTION_TYPE_QUEUE:
12246                         queue = actions->conf;
12247                         rss_desc->queue_num = 1;
12248                         rss_desc->queue[0] = queue->index;
12249                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
12250                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
12251                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
12252                         num_of_dest++;
12253                         break;
12254                 case RTE_FLOW_ACTION_TYPE_RSS:
12255                         rss = actions->conf;
12256                         memcpy(rss_desc->queue, rss->queue,
12257                                rss->queue_num * sizeof(uint16_t));
12258                         rss_desc->queue_num = rss->queue_num;
12259                         /* NULL RSS key indicates default RSS key. */
12260                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
12261                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
12262                         /*
12263                          * rss->level and rss.types should be set in advance
12264                          * when expanding items for RSS.
12265                          */
12266                         action_flags |= MLX5_FLOW_ACTION_RSS;
12267                         dev_flow->handle->fate_action = rss_desc->shared_rss ?
12268                                 MLX5_FLOW_FATE_SHARED_RSS :
12269                                 MLX5_FLOW_FATE_QUEUE;
12270                         break;
12271                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
12272                         flow->age = (uint32_t)(uintptr_t)(action->conf);
12273                         age_act = flow_aso_age_get_by_idx(dev, flow->age);
12274                         __atomic_fetch_add(&age_act->refcnt, 1,
12275                                            __ATOMIC_RELAXED);
12276                         age_act_pos = actions_n++;
12277                         action_flags |= MLX5_FLOW_ACTION_AGE;
12278                         break;
12279                 case RTE_FLOW_ACTION_TYPE_AGE:
12280                         non_shared_age = action->conf;
12281                         age_act_pos = actions_n++;
12282                         action_flags |= MLX5_FLOW_ACTION_AGE;
12283                         break;
12284                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
12285                         flow->counter = (uint32_t)(uintptr_t)(action->conf);
12286                         cnt_act = flow_dv_counter_get_by_idx(dev, flow->counter,
12287                                                              NULL);
12288                         __atomic_fetch_add(&cnt_act->shared_info.refcnt, 1,
12289                                            __ATOMIC_RELAXED);
12290                         /* Save information first, will apply later. */
12291                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12292                         break;
12293                 case RTE_FLOW_ACTION_TYPE_COUNT:
12294                         if (!dev_conf->devx) {
12295                                 return rte_flow_error_set
12296                                               (error, ENOTSUP,
12297                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12298                                                NULL,
12299                                                "count action not supported");
12300                         }
12301                         /* Save information first, will apply later. */
12302                         count = action->conf;
12303                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12304                         break;
12305                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
12306                         dev_flow->dv.actions[actions_n++] =
12307                                                 priv->sh->pop_vlan_action;
12308                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
12309                         break;
12310                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
12311                         if (!(action_flags &
12312                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
12313                                 flow_dev_get_vlan_info_from_items(items, &vlan);
12314                         vlan.eth_proto = rte_be_to_cpu_16
12315                              ((((const struct rte_flow_action_of_push_vlan *)
12316                                                    actions->conf)->ethertype));
12317                         found_action = mlx5_flow_find_action
12318                                         (actions + 1,
12319                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
12320                         if (found_action)
12321                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12322                         found_action = mlx5_flow_find_action
12323                                         (actions + 1,
12324                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
12325                         if (found_action)
12326                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12327                         if (flow_dv_create_action_push_vlan
12328                                             (dev, attr, &vlan, dev_flow, error))
12329                                 return -rte_errno;
12330                         dev_flow->dv.actions[actions_n++] =
12331                                         dev_flow->dv.push_vlan_res->action;
12332                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
12333                         break;
12334                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
12335                         /* of_vlan_push action handled this action */
12336                         MLX5_ASSERT(action_flags &
12337                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
12338                         break;
12339                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
12340                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
12341                                 break;
12342                         flow_dev_get_vlan_info_from_items(items, &vlan);
12343                         mlx5_update_vlan_vid_pcp(actions, &vlan);
12344                         /* If no VLAN push - this is a modify header action */
12345                         if (flow_dv_convert_action_modify_vlan_vid
12346                                                 (mhdr_res, actions, error))
12347                                 return -rte_errno;
12348                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
12349                         break;
12350                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
12351                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
12352                         if (flow_dv_create_action_l2_encap(dev, actions,
12353                                                            dev_flow,
12354                                                            attr->transfer,
12355                                                            error))
12356                                 return -rte_errno;
12357                         dev_flow->dv.actions[actions_n++] =
12358                                         dev_flow->dv.encap_decap->action;
12359                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12360                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12361                                 sample_act->action_flags |=
12362                                                         MLX5_FLOW_ACTION_ENCAP;
12363                         break;
12364                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
12365                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
12366                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12367                                                            attr->transfer,
12368                                                            error))
12369                                 return -rte_errno;
12370                         dev_flow->dv.actions[actions_n++] =
12371                                         dev_flow->dv.encap_decap->action;
12372                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12373                         break;
12374                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
12375                         /* Handle encap with preceding decap. */
12376                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
12377                                 if (flow_dv_create_action_raw_encap
12378                                         (dev, actions, dev_flow, attr, error))
12379                                         return -rte_errno;
12380                                 dev_flow->dv.actions[actions_n++] =
12381                                         dev_flow->dv.encap_decap->action;
12382                         } else {
12383                                 /* Handle encap without preceding decap. */
12384                                 if (flow_dv_create_action_l2_encap
12385                                     (dev, actions, dev_flow, attr->transfer,
12386                                      error))
12387                                         return -rte_errno;
12388                                 dev_flow->dv.actions[actions_n++] =
12389                                         dev_flow->dv.encap_decap->action;
12390                         }
12391                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12392                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12393                                 sample_act->action_flags |=
12394                                                         MLX5_FLOW_ACTION_ENCAP;
12395                         break;
12396                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
12397                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
12398                                 ;
12399                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
12400                                 if (flow_dv_create_action_l2_decap
12401                                     (dev, dev_flow, attr->transfer, error))
12402                                         return -rte_errno;
12403                                 dev_flow->dv.actions[actions_n++] =
12404                                         dev_flow->dv.encap_decap->action;
12405                         }
12406                         /* If decap is followed by encap, handle it at encap. */
12407                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12408                         break;
12409                 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP:
12410                         dev_flow->dv.actions[actions_n++] =
12411                                 (void *)(uintptr_t)action->conf;
12412                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12413                         break;
12414                 case RTE_FLOW_ACTION_TYPE_JUMP:
12415                         jump_group = ((const struct rte_flow_action_jump *)
12416                                                         action->conf)->group;
12417                         grp_info.std_tbl_fix = 0;
12418                         if (dev_flow->skip_scale &
12419                                 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT))
12420                                 grp_info.skip_scale = 1;
12421                         else
12422                                 grp_info.skip_scale = 0;
12423                         ret = mlx5_flow_group_to_table(dev, tunnel,
12424                                                        jump_group,
12425                                                        &table,
12426                                                        &grp_info, error);
12427                         if (ret)
12428                                 return ret;
12429                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
12430                                                        attr->transfer,
12431                                                        !!dev_flow->external,
12432                                                        tunnel, jump_group, 0,
12433                                                        0, error);
12434                         if (!tbl)
12435                                 return rte_flow_error_set
12436                                                 (error, errno,
12437                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12438                                                  NULL,
12439                                                  "cannot create jump action.");
12440                         if (flow_dv_jump_tbl_resource_register
12441                             (dev, tbl, dev_flow, error)) {
12442                                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
12443                                 return rte_flow_error_set
12444                                                 (error, errno,
12445                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12446                                                  NULL,
12447                                                  "cannot create jump action.");
12448                         }
12449                         dev_flow->dv.actions[actions_n++] =
12450                                         dev_flow->dv.jump->action;
12451                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12452                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
12453                         sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP;
12454                         num_of_dest++;
12455                         break;
12456                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
12457                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
12458                         if (flow_dv_convert_action_modify_mac
12459                                         (mhdr_res, actions, error))
12460                                 return -rte_errno;
12461                         action_flags |= actions->type ==
12462                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
12463                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
12464                                         MLX5_FLOW_ACTION_SET_MAC_DST;
12465                         break;
12466                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
12467                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
12468                         if (flow_dv_convert_action_modify_ipv4
12469                                         (mhdr_res, actions, error))
12470                                 return -rte_errno;
12471                         action_flags |= actions->type ==
12472                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
12473                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
12474                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
12475                         break;
12476                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
12477                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
12478                         if (flow_dv_convert_action_modify_ipv6
12479                                         (mhdr_res, actions, error))
12480                                 return -rte_errno;
12481                         action_flags |= actions->type ==
12482                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
12483                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
12484                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
12485                         break;
12486                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
12487                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
12488                         if (flow_dv_convert_action_modify_tp
12489                                         (mhdr_res, actions, items,
12490                                          &flow_attr, dev_flow, !!(action_flags &
12491                                          MLX5_FLOW_ACTION_DECAP), error))
12492                                 return -rte_errno;
12493                         action_flags |= actions->type ==
12494                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
12495                                         MLX5_FLOW_ACTION_SET_TP_SRC :
12496                                         MLX5_FLOW_ACTION_SET_TP_DST;
12497                         break;
12498                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
12499                         if (flow_dv_convert_action_modify_dec_ttl
12500                                         (mhdr_res, items, &flow_attr, dev_flow,
12501                                          !!(action_flags &
12502                                          MLX5_FLOW_ACTION_DECAP), error))
12503                                 return -rte_errno;
12504                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
12505                         break;
12506                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
12507                         if (flow_dv_convert_action_modify_ttl
12508                                         (mhdr_res, actions, items, &flow_attr,
12509                                          dev_flow, !!(action_flags &
12510                                          MLX5_FLOW_ACTION_DECAP), error))
12511                                 return -rte_errno;
12512                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
12513                         break;
12514                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
12515                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
12516                         if (flow_dv_convert_action_modify_tcp_seq
12517                                         (mhdr_res, actions, error))
12518                                 return -rte_errno;
12519                         action_flags |= actions->type ==
12520                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
12521                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
12522                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
12523                         break;
12524
12525                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
12526                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
12527                         if (flow_dv_convert_action_modify_tcp_ack
12528                                         (mhdr_res, actions, error))
12529                                 return -rte_errno;
12530                         action_flags |= actions->type ==
12531                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
12532                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
12533                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
12534                         break;
12535                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
12536                         if (flow_dv_convert_action_set_reg
12537                                         (mhdr_res, actions, error))
12538                                 return -rte_errno;
12539                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12540                         break;
12541                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
12542                         if (flow_dv_convert_action_copy_mreg
12543                                         (dev, mhdr_res, actions, error))
12544                                 return -rte_errno;
12545                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12546                         break;
12547                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
12548                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
12549                         dev_flow->handle->fate_action =
12550                                         MLX5_FLOW_FATE_DEFAULT_MISS;
12551                         break;
12552                 case RTE_FLOW_ACTION_TYPE_METER:
12553                         if (!wks->fm)
12554                                 return rte_flow_error_set(error, rte_errno,
12555                                         RTE_FLOW_ERROR_TYPE_ACTION,
12556                                         NULL, "Failed to get meter in flow.");
12557                         /* Set the meter action. */
12558                         dev_flow->dv.actions[actions_n++] =
12559                                 wks->fm->meter_action;
12560                         action_flags |= MLX5_FLOW_ACTION_METER;
12561                         break;
12562                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
12563                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
12564                                                               actions, error))
12565                                 return -rte_errno;
12566                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
12567                         break;
12568                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
12569                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
12570                                                               actions, error))
12571                                 return -rte_errno;
12572                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
12573                         break;
12574                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
12575                         sample_act_pos = actions_n;
12576                         sample = (const struct rte_flow_action_sample *)
12577                                  action->conf;
12578                         actions_n++;
12579                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
12580                         /* put encap action into group if work with port id */
12581                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
12582                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
12583                                 sample_act->action_flags |=
12584                                                         MLX5_FLOW_ACTION_ENCAP;
12585                         break;
12586                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
12587                         if (flow_dv_convert_action_modify_field
12588                                         (dev, mhdr_res, actions, attr, error))
12589                                 return -rte_errno;
12590                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
12591                         break;
12592                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
12593                         owner_idx = (uint32_t)(uintptr_t)action->conf;
12594                         ct = flow_aso_ct_get_by_idx(dev, owner_idx);
12595                         if (!ct)
12596                                 return rte_flow_error_set(error, EINVAL,
12597                                                 RTE_FLOW_ERROR_TYPE_ACTION,
12598                                                 NULL,
12599                                                 "Failed to get CT object.");
12600                         if (mlx5_aso_ct_available(priv->sh, ct))
12601                                 return rte_flow_error_set(error, rte_errno,
12602                                                 RTE_FLOW_ERROR_TYPE_ACTION,
12603                                                 NULL,
12604                                                 "CT is unavailable.");
12605                         if (ct->is_original)
12606                                 dev_flow->dv.actions[actions_n] =
12607                                                         ct->dr_action_orig;
12608                         else
12609                                 dev_flow->dv.actions[actions_n] =
12610                                                         ct->dr_action_rply;
12611                         flow->indirect_type = MLX5_INDIRECT_ACTION_TYPE_CT;
12612                         flow->ct = owner_idx;
12613                         __atomic_fetch_add(&ct->refcnt, 1, __ATOMIC_RELAXED);
12614                         actions_n++;
12615                         action_flags |= MLX5_FLOW_ACTION_CT;
12616                         break;
12617                 case RTE_FLOW_ACTION_TYPE_END:
12618                         actions_end = true;
12619                         if (mhdr_res->actions_num) {
12620                                 /* create modify action if needed. */
12621                                 if (flow_dv_modify_hdr_resource_register
12622                                         (dev, mhdr_res, dev_flow, error))
12623                                         return -rte_errno;
12624                                 dev_flow->dv.actions[modify_action_position] =
12625                                         handle->dvh.modify_hdr->action;
12626                         }
12627                         /*
12628                          * Handle AGE and COUNT action by single HW counter
12629                          * when they are not shared.
12630                          */
12631                         if (action_flags & MLX5_FLOW_ACTION_AGE) {
12632                                 if ((non_shared_age &&
12633                                      count && !count->shared) ||
12634                                     !(priv->sh->flow_hit_aso_en &&
12635                                       (attr->group || attr->transfer))) {
12636                                         /* Creates age by counters. */
12637                                         cnt_act = flow_dv_prepare_counter
12638                                                                 (dev, dev_flow,
12639                                                                  flow, count,
12640                                                                  non_shared_age,
12641                                                                  error);
12642                                         if (!cnt_act)
12643                                                 return -rte_errno;
12644                                         dev_flow->dv.actions[age_act_pos] =
12645                                                                 cnt_act->action;
12646                                         break;
12647                                 }
12648                                 if (!flow->age && non_shared_age) {
12649                                         flow->age = flow_dv_aso_age_alloc
12650                                                                 (dev, error);
12651                                         if (!flow->age)
12652                                                 return -rte_errno;
12653                                         flow_dv_aso_age_params_init
12654                                                     (dev, flow->age,
12655                                                      non_shared_age->context ?
12656                                                      non_shared_age->context :
12657                                                      (void *)(uintptr_t)
12658                                                      (dev_flow->flow_idx),
12659                                                      non_shared_age->timeout);
12660                                 }
12661                                 age_act = flow_aso_age_get_by_idx(dev,
12662                                                                   flow->age);
12663                                 dev_flow->dv.actions[age_act_pos] =
12664                                                              age_act->dr_action;
12665                         }
12666                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
12667                                 /*
12668                                  * Create one count action, to be used
12669                                  * by all sub-flows.
12670                                  */
12671                                 cnt_act = flow_dv_prepare_counter(dev, dev_flow,
12672                                                                   flow, count,
12673                                                                   NULL, error);
12674                                 if (!cnt_act)
12675                                         return -rte_errno;
12676                                 dev_flow->dv.actions[actions_n++] =
12677                                                                 cnt_act->action;
12678                         }
12679                 default:
12680                         break;
12681                 }
12682                 if (mhdr_res->actions_num &&
12683                     modify_action_position == UINT32_MAX)
12684                         modify_action_position = actions_n++;
12685         }
12686         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
12687                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
12688                 int item_type = items->type;
12689
12690                 if (!mlx5_flow_os_item_supported(item_type))
12691                         return rte_flow_error_set(error, ENOTSUP,
12692                                                   RTE_FLOW_ERROR_TYPE_ITEM,
12693                                                   NULL, "item not supported");
12694                 switch (item_type) {
12695                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
12696                         flow_dv_translate_item_port_id
12697                                 (dev, match_mask, match_value, items, attr);
12698                         last_item = MLX5_FLOW_ITEM_PORT_ID;
12699                         break;
12700                 case RTE_FLOW_ITEM_TYPE_ETH:
12701                         flow_dv_translate_item_eth(match_mask, match_value,
12702                                                    items, tunnel,
12703                                                    dev_flow->dv.group);
12704                         matcher.priority = action_flags &
12705                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
12706                                         !dev_flow->external ?
12707                                         MLX5_PRIORITY_MAP_L3 :
12708                                         MLX5_PRIORITY_MAP_L2;
12709                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
12710                                              MLX5_FLOW_LAYER_OUTER_L2;
12711                         break;
12712                 case RTE_FLOW_ITEM_TYPE_VLAN:
12713                         flow_dv_translate_item_vlan(dev_flow,
12714                                                     match_mask, match_value,
12715                                                     items, tunnel,
12716                                                     dev_flow->dv.group);
12717                         matcher.priority = MLX5_PRIORITY_MAP_L2;
12718                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
12719                                               MLX5_FLOW_LAYER_INNER_VLAN) :
12720                                              (MLX5_FLOW_LAYER_OUTER_L2 |
12721                                               MLX5_FLOW_LAYER_OUTER_VLAN);
12722                         break;
12723                 case RTE_FLOW_ITEM_TYPE_IPV4:
12724                         mlx5_flow_tunnel_ip_check(items, next_protocol,
12725                                                   &item_flags, &tunnel);
12726                         flow_dv_translate_item_ipv4(match_mask, match_value,
12727                                                     items, tunnel,
12728                                                     dev_flow->dv.group);
12729                         matcher.priority = MLX5_PRIORITY_MAP_L3;
12730                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
12731                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
12732                         if (items->mask != NULL &&
12733                             ((const struct rte_flow_item_ipv4 *)
12734                              items->mask)->hdr.next_proto_id) {
12735                                 next_protocol =
12736                                         ((const struct rte_flow_item_ipv4 *)
12737                                          (items->spec))->hdr.next_proto_id;
12738                                 next_protocol &=
12739                                         ((const struct rte_flow_item_ipv4 *)
12740                                          (items->mask))->hdr.next_proto_id;
12741                         } else {
12742                                 /* Reset for inner layer. */
12743                                 next_protocol = 0xff;
12744                         }
12745                         break;
12746                 case RTE_FLOW_ITEM_TYPE_IPV6:
12747                         mlx5_flow_tunnel_ip_check(items, next_protocol,
12748                                                   &item_flags, &tunnel);
12749                         flow_dv_translate_item_ipv6(match_mask, match_value,
12750                                                     items, tunnel,
12751                                                     dev_flow->dv.group);
12752                         matcher.priority = MLX5_PRIORITY_MAP_L3;
12753                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
12754                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
12755                         if (items->mask != NULL &&
12756                             ((const struct rte_flow_item_ipv6 *)
12757                              items->mask)->hdr.proto) {
12758                                 next_protocol =
12759                                         ((const struct rte_flow_item_ipv6 *)
12760                                          items->spec)->hdr.proto;
12761                                 next_protocol &=
12762                                         ((const struct rte_flow_item_ipv6 *)
12763                                          items->mask)->hdr.proto;
12764                         } else {
12765                                 /* Reset for inner layer. */
12766                                 next_protocol = 0xff;
12767                         }
12768                         break;
12769                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
12770                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
12771                                                              match_value,
12772                                                              items, tunnel);
12773                         last_item = tunnel ?
12774                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
12775                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
12776                         if (items->mask != NULL &&
12777                             ((const struct rte_flow_item_ipv6_frag_ext *)
12778                              items->mask)->hdr.next_header) {
12779                                 next_protocol =
12780                                 ((const struct rte_flow_item_ipv6_frag_ext *)
12781                                  items->spec)->hdr.next_header;
12782                                 next_protocol &=
12783                                 ((const struct rte_flow_item_ipv6_frag_ext *)
12784                                  items->mask)->hdr.next_header;
12785                         } else {
12786                                 /* Reset for inner layer. */
12787                                 next_protocol = 0xff;
12788                         }
12789                         break;
12790                 case RTE_FLOW_ITEM_TYPE_TCP:
12791                         flow_dv_translate_item_tcp(match_mask, match_value,
12792                                                    items, tunnel);
12793                         matcher.priority = MLX5_PRIORITY_MAP_L4;
12794                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
12795                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
12796                         break;
12797                 case RTE_FLOW_ITEM_TYPE_UDP:
12798                         flow_dv_translate_item_udp(match_mask, match_value,
12799                                                    items, tunnel);
12800                         matcher.priority = MLX5_PRIORITY_MAP_L4;
12801                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
12802                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
12803                         break;
12804                 case RTE_FLOW_ITEM_TYPE_GRE:
12805                         flow_dv_translate_item_gre(match_mask, match_value,
12806                                                    items, tunnel);
12807                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12808                         last_item = MLX5_FLOW_LAYER_GRE;
12809                         break;
12810                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
12811                         flow_dv_translate_item_gre_key(match_mask,
12812                                                        match_value, items);
12813                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
12814                         break;
12815                 case RTE_FLOW_ITEM_TYPE_NVGRE:
12816                         flow_dv_translate_item_nvgre(match_mask, match_value,
12817                                                      items, tunnel);
12818                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12819                         last_item = MLX5_FLOW_LAYER_GRE;
12820                         break;
12821                 case RTE_FLOW_ITEM_TYPE_VXLAN:
12822                         flow_dv_translate_item_vxlan(match_mask, match_value,
12823                                                      items, tunnel);
12824                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12825                         last_item = MLX5_FLOW_LAYER_VXLAN;
12826                         break;
12827                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
12828                         flow_dv_translate_item_vxlan_gpe(match_mask,
12829                                                          match_value, items,
12830                                                          tunnel);
12831                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12832                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
12833                         break;
12834                 case RTE_FLOW_ITEM_TYPE_GENEVE:
12835                         flow_dv_translate_item_geneve(match_mask, match_value,
12836                                                       items, tunnel);
12837                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12838                         last_item = MLX5_FLOW_LAYER_GENEVE;
12839                         break;
12840                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
12841                         ret = flow_dv_translate_item_geneve_opt(dev, match_mask,
12842                                                           match_value,
12843                                                           items, error);
12844                         if (ret)
12845                                 return rte_flow_error_set(error, -ret,
12846                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
12847                                         "cannot create GENEVE TLV option");
12848                         flow->geneve_tlv_option = 1;
12849                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
12850                         break;
12851                 case RTE_FLOW_ITEM_TYPE_MPLS:
12852                         flow_dv_translate_item_mpls(match_mask, match_value,
12853                                                     items, last_item, tunnel);
12854                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12855                         last_item = MLX5_FLOW_LAYER_MPLS;
12856                         break;
12857                 case RTE_FLOW_ITEM_TYPE_MARK:
12858                         flow_dv_translate_item_mark(dev, match_mask,
12859                                                     match_value, items);
12860                         last_item = MLX5_FLOW_ITEM_MARK;
12861                         break;
12862                 case RTE_FLOW_ITEM_TYPE_META:
12863                         flow_dv_translate_item_meta(dev, match_mask,
12864                                                     match_value, attr, items);
12865                         last_item = MLX5_FLOW_ITEM_METADATA;
12866                         break;
12867                 case RTE_FLOW_ITEM_TYPE_ICMP:
12868                         flow_dv_translate_item_icmp(match_mask, match_value,
12869                                                     items, tunnel);
12870                         last_item = MLX5_FLOW_LAYER_ICMP;
12871                         break;
12872                 case RTE_FLOW_ITEM_TYPE_ICMP6:
12873                         flow_dv_translate_item_icmp6(match_mask, match_value,
12874                                                       items, tunnel);
12875                         last_item = MLX5_FLOW_LAYER_ICMP6;
12876                         break;
12877                 case RTE_FLOW_ITEM_TYPE_TAG:
12878                         flow_dv_translate_item_tag(dev, match_mask,
12879                                                    match_value, items);
12880                         last_item = MLX5_FLOW_ITEM_TAG;
12881                         break;
12882                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
12883                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
12884                                                         match_value, items);
12885                         last_item = MLX5_FLOW_ITEM_TAG;
12886                         break;
12887                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
12888                         flow_dv_translate_item_tx_queue(dev, match_mask,
12889                                                         match_value,
12890                                                         items);
12891                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
12892                         break;
12893                 case RTE_FLOW_ITEM_TYPE_GTP:
12894                         flow_dv_translate_item_gtp(match_mask, match_value,
12895                                                    items, tunnel);
12896                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12897                         last_item = MLX5_FLOW_LAYER_GTP;
12898                         break;
12899                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
12900                         ret = flow_dv_translate_item_gtp_psc(match_mask,
12901                                                           match_value,
12902                                                           items);
12903                         if (ret)
12904                                 return rte_flow_error_set(error, -ret,
12905                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
12906                                         "cannot create GTP PSC item");
12907                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
12908                         break;
12909                 case RTE_FLOW_ITEM_TYPE_ECPRI:
12910                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
12911                                 /* Create it only the first time to be used. */
12912                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
12913                                 if (ret)
12914                                         return rte_flow_error_set
12915                                                 (error, -ret,
12916                                                 RTE_FLOW_ERROR_TYPE_ITEM,
12917                                                 NULL,
12918                                                 "cannot create eCPRI parser");
12919                         }
12920                         /* Adjust the length matcher and device flow value. */
12921                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
12922                         dev_flow->dv.value.size =
12923                                         MLX5_ST_SZ_BYTES(fte_match_param);
12924                         flow_dv_translate_item_ecpri(dev, match_mask,
12925                                                      match_value, items);
12926                         /* No other protocol should follow eCPRI layer. */
12927                         last_item = MLX5_FLOW_LAYER_ECPRI;
12928                         break;
12929                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
12930                         flow_dv_translate_item_integrity(match_mask,
12931                                                          match_value,
12932                                                          head_item, items);
12933                         break;
12934                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
12935                         flow_dv_translate_item_aso_ct(dev, match_mask,
12936                                                       match_value, items);
12937                         break;
12938                 default:
12939                         break;
12940                 }
12941                 item_flags |= last_item;
12942         }
12943         /*
12944          * When E-Switch mode is enabled, we have two cases where we need to
12945          * set the source port manually.
12946          * The first one, is in case of Nic steering rule, and the second is
12947          * E-Switch rule where no port_id item was found. In both cases
12948          * the source port is set according the current port in use.
12949          */
12950         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
12951             (priv->representor || priv->master)) {
12952                 if (flow_dv_translate_item_port_id(dev, match_mask,
12953                                                    match_value, NULL, attr))
12954                         return -rte_errno;
12955         }
12956 #ifdef RTE_LIBRTE_MLX5_DEBUG
12957         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
12958                                               dev_flow->dv.value.buf));
12959 #endif
12960         /*
12961          * Layers may be already initialized from prefix flow if this dev_flow
12962          * is the suffix flow.
12963          */
12964         handle->layers |= item_flags;
12965         if (action_flags & MLX5_FLOW_ACTION_RSS)
12966                 flow_dv_hashfields_set(dev_flow, rss_desc);
12967         /* If has RSS action in the sample action, the Sample/Mirror resource
12968          * should be registered after the hash filed be update.
12969          */
12970         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
12971                 ret = flow_dv_translate_action_sample(dev,
12972                                                       sample,
12973                                                       dev_flow, attr,
12974                                                       &num_of_dest,
12975                                                       sample_actions,
12976                                                       &sample_res,
12977                                                       error);
12978                 if (ret < 0)
12979                         return ret;
12980                 ret = flow_dv_create_action_sample(dev,
12981                                                    dev_flow,
12982                                                    num_of_dest,
12983                                                    &sample_res,
12984                                                    &mdest_res,
12985                                                    sample_actions,
12986                                                    action_flags,
12987                                                    error);
12988                 if (ret < 0)
12989                         return rte_flow_error_set
12990                                                 (error, rte_errno,
12991                                                 RTE_FLOW_ERROR_TYPE_ACTION,
12992                                                 NULL,
12993                                                 "cannot create sample action");
12994                 if (num_of_dest > 1) {
12995                         dev_flow->dv.actions[sample_act_pos] =
12996                         dev_flow->dv.dest_array_res->action;
12997                 } else {
12998                         dev_flow->dv.actions[sample_act_pos] =
12999                         dev_flow->dv.sample_res->verbs_action;
13000                 }
13001         }
13002         /*
13003          * For multiple destination (sample action with ratio=1), the encap
13004          * action and port id action will be combined into group action.
13005          * So need remove the original these actions in the flow and only
13006          * use the sample action instead of.
13007          */
13008         if (num_of_dest > 1 &&
13009             (sample_act->dr_port_id_action || sample_act->dr_jump_action)) {
13010                 int i;
13011                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
13012
13013                 for (i = 0; i < actions_n; i++) {
13014                         if ((sample_act->dr_encap_action &&
13015                                 sample_act->dr_encap_action ==
13016                                 dev_flow->dv.actions[i]) ||
13017                                 (sample_act->dr_port_id_action &&
13018                                 sample_act->dr_port_id_action ==
13019                                 dev_flow->dv.actions[i]) ||
13020                                 (sample_act->dr_jump_action &&
13021                                 sample_act->dr_jump_action ==
13022                                 dev_flow->dv.actions[i]))
13023                                 continue;
13024                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
13025                 }
13026                 memcpy((void *)dev_flow->dv.actions,
13027                                 (void *)temp_actions,
13028                                 tmp_actions_n * sizeof(void *));
13029                 actions_n = tmp_actions_n;
13030         }
13031         dev_flow->dv.actions_n = actions_n;
13032         dev_flow->act_flags = action_flags;
13033         if (wks->skip_matcher_reg)
13034                 return 0;
13035         /* Register matcher. */
13036         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
13037                                     matcher.mask.size);
13038         matcher.priority = mlx5_get_matcher_priority(dev, attr,
13039                                         matcher.priority);
13040         /* reserved field no needs to be set to 0 here. */
13041         tbl_key.is_fdb = attr->transfer;
13042         tbl_key.is_egress = attr->egress;
13043         tbl_key.level = dev_flow->dv.group;
13044         tbl_key.id = dev_flow->dv.table_id;
13045         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
13046                                      tunnel, attr->group, error))
13047                 return -rte_errno;
13048         return 0;
13049 }
13050
13051 /**
13052  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13053  * and tunnel.
13054  *
13055  * @param[in, out] action
13056  *   Shred RSS action holding hash RX queue objects.
13057  * @param[in] hash_fields
13058  *   Defines combination of packet fields to participate in RX hash.
13059  * @param[in] tunnel
13060  *   Tunnel type
13061  * @param[in] hrxq_idx
13062  *   Hash RX queue index to set.
13063  *
13064  * @return
13065  *   0 on success, otherwise negative errno value.
13066  */
13067 static int
13068 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
13069                               const uint64_t hash_fields,
13070                               uint32_t hrxq_idx)
13071 {
13072         uint32_t *hrxqs = action->hrxq;
13073
13074         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13075         case MLX5_RSS_HASH_IPV4:
13076                 /* fall-through. */
13077         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13078                 /* fall-through. */
13079         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13080                 hrxqs[0] = hrxq_idx;
13081                 return 0;
13082         case MLX5_RSS_HASH_IPV4_TCP:
13083                 /* fall-through. */
13084         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13085                 /* fall-through. */
13086         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13087                 hrxqs[1] = hrxq_idx;
13088                 return 0;
13089         case MLX5_RSS_HASH_IPV4_UDP:
13090                 /* fall-through. */
13091         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13092                 /* fall-through. */
13093         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13094                 hrxqs[2] = hrxq_idx;
13095                 return 0;
13096         case MLX5_RSS_HASH_IPV6:
13097                 /* fall-through. */
13098         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13099                 /* fall-through. */
13100         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13101                 hrxqs[3] = hrxq_idx;
13102                 return 0;
13103         case MLX5_RSS_HASH_IPV6_TCP:
13104                 /* fall-through. */
13105         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13106                 /* fall-through. */
13107         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13108                 hrxqs[4] = hrxq_idx;
13109                 return 0;
13110         case MLX5_RSS_HASH_IPV6_UDP:
13111                 /* fall-through. */
13112         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13113                 /* fall-through. */
13114         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13115                 hrxqs[5] = hrxq_idx;
13116                 return 0;
13117         case MLX5_RSS_HASH_NONE:
13118                 hrxqs[6] = hrxq_idx;
13119                 return 0;
13120         default:
13121                 return -1;
13122         }
13123 }
13124
13125 /**
13126  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13127  * and tunnel.
13128  *
13129  * @param[in] dev
13130  *   Pointer to the Ethernet device structure.
13131  * @param[in] idx
13132  *   Shared RSS action ID holding hash RX queue objects.
13133  * @param[in] hash_fields
13134  *   Defines combination of packet fields to participate in RX hash.
13135  * @param[in] tunnel
13136  *   Tunnel type
13137  *
13138  * @return
13139  *   Valid hash RX queue index, otherwise 0.
13140  */
13141 static uint32_t
13142 __flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
13143                                  const uint64_t hash_fields)
13144 {
13145         struct mlx5_priv *priv = dev->data->dev_private;
13146         struct mlx5_shared_action_rss *shared_rss =
13147             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
13148         const uint32_t *hrxqs = shared_rss->hrxq;
13149
13150         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13151         case MLX5_RSS_HASH_IPV4:
13152                 /* fall-through. */
13153         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13154                 /* fall-through. */
13155         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13156                 return hrxqs[0];
13157         case MLX5_RSS_HASH_IPV4_TCP:
13158                 /* fall-through. */
13159         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13160                 /* fall-through. */
13161         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13162                 return hrxqs[1];
13163         case MLX5_RSS_HASH_IPV4_UDP:
13164                 /* fall-through. */
13165         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13166                 /* fall-through. */
13167         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13168                 return hrxqs[2];
13169         case MLX5_RSS_HASH_IPV6:
13170                 /* fall-through. */
13171         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13172                 /* fall-through. */
13173         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13174                 return hrxqs[3];
13175         case MLX5_RSS_HASH_IPV6_TCP:
13176                 /* fall-through. */
13177         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13178                 /* fall-through. */
13179         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13180                 return hrxqs[4];
13181         case MLX5_RSS_HASH_IPV6_UDP:
13182                 /* fall-through. */
13183         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13184                 /* fall-through. */
13185         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13186                 return hrxqs[5];
13187         case MLX5_RSS_HASH_NONE:
13188                 return hrxqs[6];
13189         default:
13190                 return 0;
13191         }
13192
13193 }
13194
13195 /**
13196  * Apply the flow to the NIC, lock free,
13197  * (mutex should be acquired by caller).
13198  *
13199  * @param[in] dev
13200  *   Pointer to the Ethernet device structure.
13201  * @param[in, out] flow
13202  *   Pointer to flow structure.
13203  * @param[out] error
13204  *   Pointer to error structure.
13205  *
13206  * @return
13207  *   0 on success, a negative errno value otherwise and rte_errno is set.
13208  */
13209 static int
13210 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
13211               struct rte_flow_error *error)
13212 {
13213         struct mlx5_flow_dv_workspace *dv;
13214         struct mlx5_flow_handle *dh;
13215         struct mlx5_flow_handle_dv *dv_h;
13216         struct mlx5_flow *dev_flow;
13217         struct mlx5_priv *priv = dev->data->dev_private;
13218         uint32_t handle_idx;
13219         int n;
13220         int err;
13221         int idx;
13222         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
13223         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
13224
13225         MLX5_ASSERT(wks);
13226         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
13227                 dev_flow = &wks->flows[idx];
13228                 dv = &dev_flow->dv;
13229                 dh = dev_flow->handle;
13230                 dv_h = &dh->dvh;
13231                 n = dv->actions_n;
13232                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
13233                         if (dv->transfer) {
13234                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13235                                 dv->actions[n++] = priv->sh->dr_drop_action;
13236                         } else {
13237 #ifdef HAVE_MLX5DV_DR
13238                                 /* DR supports drop action placeholder. */
13239                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13240                                 dv->actions[n++] = priv->sh->dr_drop_action;
13241 #else
13242                                 /* For DV we use the explicit drop queue. */
13243                                 MLX5_ASSERT(priv->drop_queue.hrxq);
13244                                 dv->actions[n++] =
13245                                                 priv->drop_queue.hrxq->action;
13246 #endif
13247                         }
13248                 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
13249                            !dv_h->rix_sample && !dv_h->rix_dest_array)) {
13250                         struct mlx5_hrxq *hrxq;
13251                         uint32_t hrxq_idx;
13252
13253                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc,
13254                                                     &hrxq_idx);
13255                         if (!hrxq) {
13256                                 rte_flow_error_set
13257                                         (error, rte_errno,
13258                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13259                                          "cannot get hash queue");
13260                                 goto error;
13261                         }
13262                         dh->rix_hrxq = hrxq_idx;
13263                         dv->actions[n++] = hrxq->action;
13264                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13265                         struct mlx5_hrxq *hrxq = NULL;
13266                         uint32_t hrxq_idx;
13267
13268                         hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev,
13269                                                 rss_desc->shared_rss,
13270                                                 dev_flow->hash_fields);
13271                         if (hrxq_idx)
13272                                 hrxq = mlx5_ipool_get
13273                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
13274                                          hrxq_idx);
13275                         if (!hrxq) {
13276                                 rte_flow_error_set
13277                                         (error, rte_errno,
13278                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13279                                          "cannot get hash queue");
13280                                 goto error;
13281                         }
13282                         dh->rix_srss = rss_desc->shared_rss;
13283                         dv->actions[n++] = hrxq->action;
13284                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
13285                         if (!priv->sh->default_miss_action) {
13286                                 rte_flow_error_set
13287                                         (error, rte_errno,
13288                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13289                                          "default miss action not be created.");
13290                                 goto error;
13291                         }
13292                         dv->actions[n++] = priv->sh->default_miss_action;
13293                 }
13294                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
13295                                                (void *)&dv->value, n,
13296                                                dv->actions, &dh->drv_flow);
13297                 if (err) {
13298                         rte_flow_error_set(error, errno,
13299                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
13300                                            NULL,
13301                                            "hardware refuses to create flow");
13302                         goto error;
13303                 }
13304                 if (priv->vmwa_context &&
13305                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
13306                         /*
13307                          * The rule contains the VLAN pattern.
13308                          * For VF we are going to create VLAN
13309                          * interface to make hypervisor set correct
13310                          * e-Switch vport context.
13311                          */
13312                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
13313                 }
13314         }
13315         return 0;
13316 error:
13317         err = rte_errno; /* Save rte_errno before cleanup. */
13318         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
13319                        handle_idx, dh, next) {
13320                 /* hrxq is union, don't clear it if the flag is not set. */
13321                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
13322                         mlx5_hrxq_release(dev, dh->rix_hrxq);
13323                         dh->rix_hrxq = 0;
13324                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13325                         dh->rix_srss = 0;
13326                 }
13327                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
13328                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
13329         }
13330         rte_errno = err; /* Restore rte_errno. */
13331         return -rte_errno;
13332 }
13333
13334 void
13335 flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused,
13336                           struct mlx5_cache_entry *entry)
13337 {
13338         struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache),
13339                                                           entry);
13340
13341         claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object));
13342         mlx5_free(cache);
13343 }
13344
13345 /**
13346  * Release the flow matcher.
13347  *
13348  * @param dev
13349  *   Pointer to Ethernet device.
13350  * @param port_id
13351  *   Index to port ID action resource.
13352  *
13353  * @return
13354  *   1 while a reference on it exists, 0 when freed.
13355  */
13356 static int
13357 flow_dv_matcher_release(struct rte_eth_dev *dev,
13358                         struct mlx5_flow_handle *handle)
13359 {
13360         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
13361         struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl,
13362                                                             typeof(*tbl), tbl);
13363         int ret;
13364
13365         MLX5_ASSERT(matcher->matcher_object);
13366         ret = mlx5_cache_unregister(&tbl->matchers, &matcher->entry);
13367         flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
13368         return ret;
13369 }
13370
13371 /**
13372  * Release encap_decap resource.
13373  *
13374  * @param list
13375  *   Pointer to the hash list.
13376  * @param entry
13377  *   Pointer to exist resource entry object.
13378  */
13379 void
13380 flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
13381                               struct mlx5_hlist_entry *entry)
13382 {
13383         struct mlx5_dev_ctx_shared *sh = list->ctx;
13384         struct mlx5_flow_dv_encap_decap_resource *res =
13385                 container_of(entry, typeof(*res), entry);
13386
13387         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13388         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
13389 }
13390
13391 /**
13392  * Release an encap/decap resource.
13393  *
13394  * @param dev
13395  *   Pointer to Ethernet device.
13396  * @param encap_decap_idx
13397  *   Index of encap decap resource.
13398  *
13399  * @return
13400  *   1 while a reference on it exists, 0 when freed.
13401  */
13402 static int
13403 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
13404                                      uint32_t encap_decap_idx)
13405 {
13406         struct mlx5_priv *priv = dev->data->dev_private;
13407         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
13408
13409         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
13410                                         encap_decap_idx);
13411         if (!cache_resource)
13412                 return 0;
13413         MLX5_ASSERT(cache_resource->action);
13414         return mlx5_hlist_unregister(priv->sh->encaps_decaps,
13415                                      &cache_resource->entry);
13416 }
13417
13418 /**
13419  * Release an jump to table action resource.
13420  *
13421  * @param dev
13422  *   Pointer to Ethernet device.
13423  * @param rix_jump
13424  *   Index to the jump action resource.
13425  *
13426  * @return
13427  *   1 while a reference on it exists, 0 when freed.
13428  */
13429 static int
13430 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
13431                                   uint32_t rix_jump)
13432 {
13433         struct mlx5_priv *priv = dev->data->dev_private;
13434         struct mlx5_flow_tbl_data_entry *tbl_data;
13435
13436         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
13437                                   rix_jump);
13438         if (!tbl_data)
13439                 return 0;
13440         return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl);
13441 }
13442
13443 void
13444 flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused,
13445                          struct mlx5_hlist_entry *entry)
13446 {
13447         struct mlx5_flow_dv_modify_hdr_resource *res =
13448                 container_of(entry, typeof(*res), entry);
13449
13450         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13451         mlx5_free(entry);
13452 }
13453
13454 /**
13455  * Release a modify-header resource.
13456  *
13457  * @param dev
13458  *   Pointer to Ethernet device.
13459  * @param handle
13460  *   Pointer to mlx5_flow_handle.
13461  *
13462  * @return
13463  *   1 while a reference on it exists, 0 when freed.
13464  */
13465 static int
13466 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
13467                                     struct mlx5_flow_handle *handle)
13468 {
13469         struct mlx5_priv *priv = dev->data->dev_private;
13470         struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr;
13471
13472         MLX5_ASSERT(entry->action);
13473         return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry);
13474 }
13475
13476 void
13477 flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
13478                           struct mlx5_cache_entry *entry)
13479 {
13480         struct mlx5_dev_ctx_shared *sh = list->ctx;
13481         struct mlx5_flow_dv_port_id_action_resource *cache =
13482                         container_of(entry, typeof(*cache), entry);
13483
13484         claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
13485         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], cache->idx);
13486 }
13487
13488 /**
13489  * Release port ID action resource.
13490  *
13491  * @param dev
13492  *   Pointer to Ethernet device.
13493  * @param handle
13494  *   Pointer to mlx5_flow_handle.
13495  *
13496  * @return
13497  *   1 while a reference on it exists, 0 when freed.
13498  */
13499 static int
13500 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
13501                                         uint32_t port_id)
13502 {
13503         struct mlx5_priv *priv = dev->data->dev_private;
13504         struct mlx5_flow_dv_port_id_action_resource *cache;
13505
13506         cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
13507         if (!cache)
13508                 return 0;
13509         MLX5_ASSERT(cache->action);
13510         return mlx5_cache_unregister(&priv->sh->port_id_action_list,
13511                                      &cache->entry);
13512 }
13513
13514 /**
13515  * Release shared RSS action resource.
13516  *
13517  * @param dev
13518  *   Pointer to Ethernet device.
13519  * @param srss
13520  *   Shared RSS action index.
13521  */
13522 static void
13523 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
13524 {
13525         struct mlx5_priv *priv = dev->data->dev_private;
13526         struct mlx5_shared_action_rss *shared_rss;
13527
13528         shared_rss = mlx5_ipool_get
13529                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss);
13530         __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
13531 }
13532
13533 void
13534 flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
13535                             struct mlx5_cache_entry *entry)
13536 {
13537         struct mlx5_dev_ctx_shared *sh = list->ctx;
13538         struct mlx5_flow_dv_push_vlan_action_resource *cache =
13539                         container_of(entry, typeof(*cache), entry);
13540
13541         claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
13542         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], cache->idx);
13543 }
13544
13545 /**
13546  * Release push vlan action resource.
13547  *
13548  * @param dev
13549  *   Pointer to Ethernet device.
13550  * @param handle
13551  *   Pointer to mlx5_flow_handle.
13552  *
13553  * @return
13554  *   1 while a reference on it exists, 0 when freed.
13555  */
13556 static int
13557 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
13558                                           struct mlx5_flow_handle *handle)
13559 {
13560         struct mlx5_priv *priv = dev->data->dev_private;
13561         struct mlx5_flow_dv_push_vlan_action_resource *cache;
13562         uint32_t idx = handle->dvh.rix_push_vlan;
13563
13564         cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
13565         if (!cache)
13566                 return 0;
13567         MLX5_ASSERT(cache->action);
13568         return mlx5_cache_unregister(&priv->sh->push_vlan_action_list,
13569                                      &cache->entry);
13570 }
13571
13572 /**
13573  * Release the fate resource.
13574  *
13575  * @param dev
13576  *   Pointer to Ethernet device.
13577  * @param handle
13578  *   Pointer to mlx5_flow_handle.
13579  */
13580 static void
13581 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
13582                                struct mlx5_flow_handle *handle)
13583 {
13584         if (!handle->rix_fate)
13585                 return;
13586         switch (handle->fate_action) {
13587         case MLX5_FLOW_FATE_QUEUE:
13588                 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array)
13589                         mlx5_hrxq_release(dev, handle->rix_hrxq);
13590                 break;
13591         case MLX5_FLOW_FATE_JUMP:
13592                 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump);
13593                 break;
13594         case MLX5_FLOW_FATE_PORT_ID:
13595                 flow_dv_port_id_action_resource_release(dev,
13596                                 handle->rix_port_id_action);
13597                 break;
13598         default:
13599                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
13600                 break;
13601         }
13602         handle->rix_fate = 0;
13603 }
13604
13605 void
13606 flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused,
13607                          struct mlx5_cache_entry *entry)
13608 {
13609         struct mlx5_flow_dv_sample_resource *cache_resource =
13610                         container_of(entry, typeof(*cache_resource), entry);
13611         struct rte_eth_dev *dev = cache_resource->dev;
13612         struct mlx5_priv *priv = dev->data->dev_private;
13613
13614         if (cache_resource->verbs_action)
13615                 claim_zero(mlx5_flow_os_destroy_flow_action
13616                                 (cache_resource->verbs_action));
13617         if (cache_resource->normal_path_tbl)
13618                 flow_dv_tbl_resource_release(MLX5_SH(dev),
13619                         cache_resource->normal_path_tbl);
13620         flow_dv_sample_sub_actions_release(dev,
13621                                 &cache_resource->sample_idx);
13622         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
13623                         cache_resource->idx);
13624         DRV_LOG(DEBUG, "sample resource %p: removed",
13625                 (void *)cache_resource);
13626 }
13627
13628 /**
13629  * Release an sample resource.
13630  *
13631  * @param dev
13632  *   Pointer to Ethernet device.
13633  * @param handle
13634  *   Pointer to mlx5_flow_handle.
13635  *
13636  * @return
13637  *   1 while a reference on it exists, 0 when freed.
13638  */
13639 static int
13640 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
13641                                      struct mlx5_flow_handle *handle)
13642 {
13643         struct mlx5_priv *priv = dev->data->dev_private;
13644         struct mlx5_flow_dv_sample_resource *cache_resource;
13645
13646         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
13647                          handle->dvh.rix_sample);
13648         if (!cache_resource)
13649                 return 0;
13650         MLX5_ASSERT(cache_resource->verbs_action);
13651         return mlx5_cache_unregister(&priv->sh->sample_action_list,
13652                                      &cache_resource->entry);
13653 }
13654
13655 void
13656 flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused,
13657                              struct mlx5_cache_entry *entry)
13658 {
13659         struct mlx5_flow_dv_dest_array_resource *cache_resource =
13660                         container_of(entry, typeof(*cache_resource), entry);
13661         struct rte_eth_dev *dev = cache_resource->dev;
13662         struct mlx5_priv *priv = dev->data->dev_private;
13663         uint32_t i = 0;
13664
13665         MLX5_ASSERT(cache_resource->action);
13666         if (cache_resource->action)
13667                 claim_zero(mlx5_flow_os_destroy_flow_action
13668                                         (cache_resource->action));
13669         for (; i < cache_resource->num_of_dest; i++)
13670                 flow_dv_sample_sub_actions_release(dev,
13671                                 &cache_resource->sample_idx[i]);
13672         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
13673                         cache_resource->idx);
13674         DRV_LOG(DEBUG, "destination array resource %p: removed",
13675                 (void *)cache_resource);
13676 }
13677
13678 /**
13679  * Release an destination array resource.
13680  *
13681  * @param dev
13682  *   Pointer to Ethernet device.
13683  * @param handle
13684  *   Pointer to mlx5_flow_handle.
13685  *
13686  * @return
13687  *   1 while a reference on it exists, 0 when freed.
13688  */
13689 static int
13690 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
13691                                     struct mlx5_flow_handle *handle)
13692 {
13693         struct mlx5_priv *priv = dev->data->dev_private;
13694         struct mlx5_flow_dv_dest_array_resource *cache;
13695
13696         cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
13697                                handle->dvh.rix_dest_array);
13698         if (!cache)
13699                 return 0;
13700         MLX5_ASSERT(cache->action);
13701         return mlx5_cache_unregister(&priv->sh->dest_array_list,
13702                                      &cache->entry);
13703 }
13704
13705 static void
13706 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev)
13707 {
13708         struct mlx5_priv *priv = dev->data->dev_private;
13709         struct mlx5_dev_ctx_shared *sh = priv->sh;
13710         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
13711                                 sh->geneve_tlv_option_resource;
13712         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
13713         if (geneve_opt_resource) {
13714                 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1,
13715                                          __ATOMIC_RELAXED))) {
13716                         claim_zero(mlx5_devx_cmd_destroy
13717                                         (geneve_opt_resource->obj));
13718                         mlx5_free(sh->geneve_tlv_option_resource);
13719                         sh->geneve_tlv_option_resource = NULL;
13720                 }
13721         }
13722         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
13723 }
13724
13725 /**
13726  * Remove the flow from the NIC but keeps it in memory.
13727  * Lock free, (mutex should be acquired by caller).
13728  *
13729  * @param[in] dev
13730  *   Pointer to Ethernet device.
13731  * @param[in, out] flow
13732  *   Pointer to flow structure.
13733  */
13734 static void
13735 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
13736 {
13737         struct mlx5_flow_handle *dh;
13738         uint32_t handle_idx;
13739         struct mlx5_priv *priv = dev->data->dev_private;
13740
13741         if (!flow)
13742                 return;
13743         handle_idx = flow->dev_handles;
13744         while (handle_idx) {
13745                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
13746                                     handle_idx);
13747                 if (!dh)
13748                         return;
13749                 if (dh->drv_flow) {
13750                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
13751                         dh->drv_flow = NULL;
13752                 }
13753                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
13754                         flow_dv_fate_resource_release(dev, dh);
13755                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
13756                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
13757                 handle_idx = dh->next.next;
13758         }
13759 }
13760
13761 /**
13762  * Remove the flow from the NIC and the memory.
13763  * Lock free, (mutex should be acquired by caller).
13764  *
13765  * @param[in] dev
13766  *   Pointer to the Ethernet device structure.
13767  * @param[in, out] flow
13768  *   Pointer to flow structure.
13769  */
13770 static void
13771 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
13772 {
13773         struct mlx5_flow_handle *dev_handle;
13774         struct mlx5_priv *priv = dev->data->dev_private;
13775         struct mlx5_flow_meter_info *fm = NULL;
13776         uint32_t srss = 0;
13777
13778         if (!flow)
13779                 return;
13780         flow_dv_remove(dev, flow);
13781         if (flow->counter) {
13782                 flow_dv_counter_free(dev, flow->counter);
13783                 flow->counter = 0;
13784         }
13785         if (flow->meter) {
13786                 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
13787                 if (fm)
13788                         mlx5_flow_meter_detach(priv, fm);
13789                 flow->meter = 0;
13790         }
13791         /* Keep the current age handling by default. */
13792         if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct)
13793                 flow_dv_aso_ct_release(dev, flow->ct);
13794         else if (flow->age)
13795                 flow_dv_aso_age_release(dev, flow->age);
13796         if (flow->geneve_tlv_option) {
13797                 flow_dv_geneve_tlv_option_resource_release(dev);
13798                 flow->geneve_tlv_option = 0;
13799         }
13800         while (flow->dev_handles) {
13801                 uint32_t tmp_idx = flow->dev_handles;
13802
13803                 dev_handle = mlx5_ipool_get(priv->sh->ipool
13804                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
13805                 if (!dev_handle)
13806                         return;
13807                 flow->dev_handles = dev_handle->next.next;
13808                 if (dev_handle->dvh.matcher)
13809                         flow_dv_matcher_release(dev, dev_handle);
13810                 if (dev_handle->dvh.rix_sample)
13811                         flow_dv_sample_resource_release(dev, dev_handle);
13812                 if (dev_handle->dvh.rix_dest_array)
13813                         flow_dv_dest_array_resource_release(dev, dev_handle);
13814                 if (dev_handle->dvh.rix_encap_decap)
13815                         flow_dv_encap_decap_resource_release(dev,
13816                                 dev_handle->dvh.rix_encap_decap);
13817                 if (dev_handle->dvh.modify_hdr)
13818                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
13819                 if (dev_handle->dvh.rix_push_vlan)
13820                         flow_dv_push_vlan_action_resource_release(dev,
13821                                                                   dev_handle);
13822                 if (dev_handle->dvh.rix_tag)
13823                         flow_dv_tag_release(dev,
13824                                             dev_handle->dvh.rix_tag);
13825                 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS)
13826                         flow_dv_fate_resource_release(dev, dev_handle);
13827                 else if (!srss)
13828                         srss = dev_handle->rix_srss;
13829                 if (fm && dev_handle->is_meter_flow_id &&
13830                     dev_handle->split_flow_id)
13831                         mlx5_ipool_free(fm->flow_ipool,
13832                                         dev_handle->split_flow_id);
13833                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
13834                            tmp_idx);
13835         }
13836         if (srss)
13837                 flow_dv_shared_rss_action_release(dev, srss);
13838 }
13839
13840 /**
13841  * Release array of hash RX queue objects.
13842  * Helper function.
13843  *
13844  * @param[in] dev
13845  *   Pointer to the Ethernet device structure.
13846  * @param[in, out] hrxqs
13847  *   Array of hash RX queue objects.
13848  *
13849  * @return
13850  *   Total number of references to hash RX queue objects in *hrxqs* array
13851  *   after this operation.
13852  */
13853 static int
13854 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
13855                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
13856 {
13857         size_t i;
13858         int remaining = 0;
13859
13860         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
13861                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
13862
13863                 if (!ret)
13864                         (*hrxqs)[i] = 0;
13865                 remaining += ret;
13866         }
13867         return remaining;
13868 }
13869
13870 /**
13871  * Release all hash RX queue objects representing shared RSS action.
13872  *
13873  * @param[in] dev
13874  *   Pointer to the Ethernet device structure.
13875  * @param[in, out] action
13876  *   Shared RSS action to remove hash RX queue objects from.
13877  *
13878  * @return
13879  *   Total number of references to hash RX queue objects stored in *action*
13880  *   after this operation.
13881  *   Expected to be 0 if no external references held.
13882  */
13883 static int
13884 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
13885                                  struct mlx5_shared_action_rss *shared_rss)
13886 {
13887         return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq);
13888 }
13889
13890 /**
13891  * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to
13892  * user input.
13893  *
13894  * Only one hash value is available for one L3+L4 combination:
13895  * for example:
13896  * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and
13897  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
13898  * same slot in mlx5_rss_hash_fields.
13899  *
13900  * @param[in] rss
13901  *   Pointer to the shared action RSS conf.
13902  * @param[in, out] hash_field
13903  *   hash_field variable needed to be adjusted.
13904  *
13905  * @return
13906  *   void
13907  */
13908 static void
13909 __flow_dv_action_rss_l34_hash_adjust(struct mlx5_shared_action_rss *rss,
13910                                      uint64_t *hash_field)
13911 {
13912         uint64_t rss_types = rss->origin.types;
13913
13914         switch (*hash_field & ~IBV_RX_HASH_INNER) {
13915         case MLX5_RSS_HASH_IPV4:
13916                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
13917                         *hash_field &= ~MLX5_RSS_HASH_IPV4;
13918                         if (rss_types & ETH_RSS_L3_DST_ONLY)
13919                                 *hash_field |= IBV_RX_HASH_DST_IPV4;
13920                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
13921                                 *hash_field |= IBV_RX_HASH_SRC_IPV4;
13922                         else
13923                                 *hash_field |= MLX5_RSS_HASH_IPV4;
13924                 }
13925                 return;
13926         case MLX5_RSS_HASH_IPV6:
13927                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
13928                         *hash_field &= ~MLX5_RSS_HASH_IPV6;
13929                         if (rss_types & ETH_RSS_L3_DST_ONLY)
13930                                 *hash_field |= IBV_RX_HASH_DST_IPV6;
13931                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
13932                                 *hash_field |= IBV_RX_HASH_SRC_IPV6;
13933                         else
13934                                 *hash_field |= MLX5_RSS_HASH_IPV6;
13935                 }
13936                 return;
13937         case MLX5_RSS_HASH_IPV4_UDP:
13938                 /* fall-through. */
13939         case MLX5_RSS_HASH_IPV6_UDP:
13940                 if (rss_types & ETH_RSS_UDP) {
13941                         *hash_field &= ~MLX5_UDP_IBV_RX_HASH;
13942                         if (rss_types & ETH_RSS_L4_DST_ONLY)
13943                                 *hash_field |= IBV_RX_HASH_DST_PORT_UDP;
13944                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
13945                                 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP;
13946                         else
13947                                 *hash_field |= MLX5_UDP_IBV_RX_HASH;
13948                 }
13949                 return;
13950         case MLX5_RSS_HASH_IPV4_TCP:
13951                 /* fall-through. */
13952         case MLX5_RSS_HASH_IPV6_TCP:
13953                 if (rss_types & ETH_RSS_TCP) {
13954                         *hash_field &= ~MLX5_TCP_IBV_RX_HASH;
13955                         if (rss_types & ETH_RSS_L4_DST_ONLY)
13956                                 *hash_field |= IBV_RX_HASH_DST_PORT_TCP;
13957                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
13958                                 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP;
13959                         else
13960                                 *hash_field |= MLX5_TCP_IBV_RX_HASH;
13961                 }
13962                 return;
13963         default:
13964                 return;
13965         }
13966 }
13967
13968 /**
13969  * Setup shared RSS action.
13970  * Prepare set of hash RX queue objects sufficient to handle all valid
13971  * hash_fields combinations (see enum ibv_rx_hash_fields).
13972  *
13973  * @param[in] dev
13974  *   Pointer to the Ethernet device structure.
13975  * @param[in] action_idx
13976  *   Shared RSS action ipool index.
13977  * @param[in, out] action
13978  *   Partially initialized shared RSS action.
13979  * @param[out] error
13980  *   Perform verbose error reporting if not NULL. Initialized in case of
13981  *   error only.
13982  *
13983  * @return
13984  *   0 on success, otherwise negative errno value.
13985  */
13986 static int
13987 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
13988                            uint32_t action_idx,
13989                            struct mlx5_shared_action_rss *shared_rss,
13990                            struct rte_flow_error *error)
13991 {
13992         struct mlx5_flow_rss_desc rss_desc = { 0 };
13993         size_t i;
13994         int err;
13995
13996         if (mlx5_ind_table_obj_setup(dev, shared_rss->ind_tbl)) {
13997                 return rte_flow_error_set(error, rte_errno,
13998                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13999                                           "cannot setup indirection table");
14000         }
14001         memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN);
14002         rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
14003         rss_desc.const_q = shared_rss->origin.queue;
14004         rss_desc.queue_num = shared_rss->origin.queue_num;
14005         /* Set non-zero value to indicate a shared RSS. */
14006         rss_desc.shared_rss = action_idx;
14007         rss_desc.ind_tbl = shared_rss->ind_tbl;
14008         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
14009                 uint32_t hrxq_idx;
14010                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
14011                 int tunnel = 0;
14012
14013                 __flow_dv_action_rss_l34_hash_adjust(shared_rss, &hash_fields);
14014                 if (shared_rss->origin.level > 1) {
14015                         hash_fields |= IBV_RX_HASH_INNER;
14016                         tunnel = 1;
14017                 }
14018                 rss_desc.tunnel = tunnel;
14019                 rss_desc.hash_fields = hash_fields;
14020                 hrxq_idx = mlx5_hrxq_get(dev, &rss_desc);
14021                 if (!hrxq_idx) {
14022                         rte_flow_error_set
14023                                 (error, rte_errno,
14024                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14025                                  "cannot get hash queue");
14026                         goto error_hrxq_new;
14027                 }
14028                 err = __flow_dv_action_rss_hrxq_set
14029                         (shared_rss, hash_fields, hrxq_idx);
14030                 MLX5_ASSERT(!err);
14031         }
14032         return 0;
14033 error_hrxq_new:
14034         err = rte_errno;
14035         __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14036         if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true))
14037                 shared_rss->ind_tbl = NULL;
14038         rte_errno = err;
14039         return -rte_errno;
14040 }
14041
14042 /**
14043  * Create shared RSS action.
14044  *
14045  * @param[in] dev
14046  *   Pointer to the Ethernet device structure.
14047  * @param[in] conf
14048  *   Shared action configuration.
14049  * @param[in] rss
14050  *   RSS action specification used to create shared action.
14051  * @param[out] error
14052  *   Perform verbose error reporting if not NULL. Initialized in case of
14053  *   error only.
14054  *
14055  * @return
14056  *   A valid shared action ID in case of success, 0 otherwise and
14057  *   rte_errno is set.
14058  */
14059 static uint32_t
14060 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
14061                             const struct rte_flow_indir_action_conf *conf,
14062                             const struct rte_flow_action_rss *rss,
14063                             struct rte_flow_error *error)
14064 {
14065         struct mlx5_priv *priv = dev->data->dev_private;
14066         struct mlx5_shared_action_rss *shared_rss = NULL;
14067         void *queue = NULL;
14068         struct rte_flow_action_rss *origin;
14069         const uint8_t *rss_key;
14070         uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
14071         uint32_t idx;
14072
14073         RTE_SET_USED(conf);
14074         queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14075                             0, SOCKET_ID_ANY);
14076         shared_rss = mlx5_ipool_zmalloc
14077                          (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx);
14078         if (!shared_rss || !queue) {
14079                 rte_flow_error_set(error, ENOMEM,
14080                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14081                                    "cannot allocate resource memory");
14082                 goto error_rss_init;
14083         }
14084         if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) {
14085                 rte_flow_error_set(error, E2BIG,
14086                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14087                                    "rss action number out of range");
14088                 goto error_rss_init;
14089         }
14090         shared_rss->ind_tbl = mlx5_malloc(MLX5_MEM_ZERO,
14091                                           sizeof(*shared_rss->ind_tbl),
14092                                           0, SOCKET_ID_ANY);
14093         if (!shared_rss->ind_tbl) {
14094                 rte_flow_error_set(error, ENOMEM,
14095                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14096                                    "cannot allocate resource memory");
14097                 goto error_rss_init;
14098         }
14099         memcpy(queue, rss->queue, queue_size);
14100         shared_rss->ind_tbl->queues = queue;
14101         shared_rss->ind_tbl->queues_n = rss->queue_num;
14102         origin = &shared_rss->origin;
14103         origin->func = rss->func;
14104         origin->level = rss->level;
14105         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
14106         origin->types = !rss->types ? ETH_RSS_IP : rss->types;
14107         /* NULL RSS key indicates default RSS key. */
14108         rss_key = !rss->key ? rss_hash_default_key : rss->key;
14109         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
14110         origin->key = &shared_rss->key[0];
14111         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
14112         origin->queue = queue;
14113         origin->queue_num = rss->queue_num;
14114         if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error))
14115                 goto error_rss_init;
14116         rte_spinlock_init(&shared_rss->action_rss_sl);
14117         __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14118         rte_spinlock_lock(&priv->shared_act_sl);
14119         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14120                      &priv->rss_shared_actions, idx, shared_rss, next);
14121         rte_spinlock_unlock(&priv->shared_act_sl);
14122         return idx;
14123 error_rss_init:
14124         if (shared_rss) {
14125                 if (shared_rss->ind_tbl)
14126                         mlx5_free(shared_rss->ind_tbl);
14127                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14128                                 idx);
14129         }
14130         if (queue)
14131                 mlx5_free(queue);
14132         return 0;
14133 }
14134
14135 /**
14136  * Destroy the shared RSS action.
14137  * Release related hash RX queue objects.
14138  *
14139  * @param[in] dev
14140  *   Pointer to the Ethernet device structure.
14141  * @param[in] idx
14142  *   The shared RSS action object ID to be removed.
14143  * @param[out] error
14144  *   Perform verbose error reporting if not NULL. Initialized in case of
14145  *   error only.
14146  *
14147  * @return
14148  *   0 on success, otherwise negative errno value.
14149  */
14150 static int
14151 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx,
14152                              struct rte_flow_error *error)
14153 {
14154         struct mlx5_priv *priv = dev->data->dev_private;
14155         struct mlx5_shared_action_rss *shared_rss =
14156             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14157         uint32_t old_refcnt = 1;
14158         int remaining;
14159         uint16_t *queue = NULL;
14160
14161         if (!shared_rss)
14162                 return rte_flow_error_set(error, EINVAL,
14163                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14164                                           "invalid shared action");
14165         remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14166         if (remaining)
14167                 return rte_flow_error_set(error, EBUSY,
14168                                           RTE_FLOW_ERROR_TYPE_ACTION,
14169                                           NULL,
14170                                           "shared rss hrxq has references");
14171         if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt,
14172                                          0, 0, __ATOMIC_ACQUIRE,
14173                                          __ATOMIC_RELAXED))
14174                 return rte_flow_error_set(error, EBUSY,
14175                                           RTE_FLOW_ERROR_TYPE_ACTION,
14176                                           NULL,
14177                                           "shared rss has references");
14178         queue = shared_rss->ind_tbl->queues;
14179         remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true);
14180         if (remaining)
14181                 return rte_flow_error_set(error, EBUSY,
14182                                           RTE_FLOW_ERROR_TYPE_ACTION,
14183                                           NULL,
14184                                           "shared rss indirection table has"
14185                                           " references");
14186         mlx5_free(queue);
14187         rte_spinlock_lock(&priv->shared_act_sl);
14188         ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14189                      &priv->rss_shared_actions, idx, shared_rss, next);
14190         rte_spinlock_unlock(&priv->shared_act_sl);
14191         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14192                         idx);
14193         return 0;
14194 }
14195
14196 /**
14197  * Create indirect action, lock free,
14198  * (mutex should be acquired by caller).
14199  * Dispatcher for action type specific call.
14200  *
14201  * @param[in] dev
14202  *   Pointer to the Ethernet device structure.
14203  * @param[in] conf
14204  *   Shared action configuration.
14205  * @param[in] action
14206  *   Action specification used to create indirect action.
14207  * @param[out] error
14208  *   Perform verbose error reporting if not NULL. Initialized in case of
14209  *   error only.
14210  *
14211  * @return
14212  *   A valid shared action handle in case of success, NULL otherwise and
14213  *   rte_errno is set.
14214  */
14215 static struct rte_flow_action_handle *
14216 flow_dv_action_create(struct rte_eth_dev *dev,
14217                       const struct rte_flow_indir_action_conf *conf,
14218                       const struct rte_flow_action *action,
14219                       struct rte_flow_error *err)
14220 {
14221         struct mlx5_priv *priv = dev->data->dev_private;
14222         uint32_t age_idx = 0;
14223         uint32_t idx = 0;
14224         uint32_t ret = 0;
14225
14226         switch (action->type) {
14227         case RTE_FLOW_ACTION_TYPE_RSS:
14228                 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err);
14229                 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS <<
14230                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14231                 break;
14232         case RTE_FLOW_ACTION_TYPE_AGE:
14233                 age_idx = flow_dv_aso_age_alloc(dev, err);
14234                 if (!age_idx) {
14235                         ret = -rte_errno;
14236                         break;
14237                 }
14238                 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
14239                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
14240                 flow_dv_aso_age_params_init(dev, age_idx,
14241                                         ((const struct rte_flow_action_age *)
14242                                                 action->conf)->context ?
14243                                         ((const struct rte_flow_action_age *)
14244                                                 action->conf)->context :
14245                                         (void *)(uintptr_t)idx,
14246                                         ((const struct rte_flow_action_age *)
14247                                                 action->conf)->timeout);
14248                 ret = age_idx;
14249                 break;
14250         case RTE_FLOW_ACTION_TYPE_COUNT:
14251                 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL);
14252                 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT <<
14253                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14254                 break;
14255         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
14256                 ret = flow_dv_translate_create_conntrack(dev, action->conf,
14257                                                          err);
14258                 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret);
14259                 break;
14260         default:
14261                 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
14262                                    NULL, "action type not supported");
14263                 break;
14264         }
14265         return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL;
14266 }
14267
14268 /**
14269  * Destroy the indirect action.
14270  * Release action related resources on the NIC and the memory.
14271  * Lock free, (mutex should be acquired by caller).
14272  * Dispatcher for action type specific call.
14273  *
14274  * @param[in] dev
14275  *   Pointer to the Ethernet device structure.
14276  * @param[in] handle
14277  *   The indirect action object handle to be removed.
14278  * @param[out] error
14279  *   Perform verbose error reporting if not NULL. Initialized in case of
14280  *   error only.
14281  *
14282  * @return
14283  *   0 on success, otherwise negative errno value.
14284  */
14285 static int
14286 flow_dv_action_destroy(struct rte_eth_dev *dev,
14287                        struct rte_flow_action_handle *handle,
14288                        struct rte_flow_error *error)
14289 {
14290         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14291         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14292         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14293         struct mlx5_flow_counter *cnt;
14294         uint32_t no_flow_refcnt = 1;
14295         int ret;
14296
14297         switch (type) {
14298         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14299                 return __flow_dv_action_rss_release(dev, idx, error);
14300         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
14301                 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL);
14302                 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt,
14303                                                  &no_flow_refcnt, 1, false,
14304                                                  __ATOMIC_ACQUIRE,
14305                                                  __ATOMIC_RELAXED))
14306                         return rte_flow_error_set(error, EBUSY,
14307                                                   RTE_FLOW_ERROR_TYPE_ACTION,
14308                                                   NULL,
14309                                                   "Indirect count action has references");
14310                 flow_dv_counter_free(dev, idx);
14311                 return 0;
14312         case MLX5_INDIRECT_ACTION_TYPE_AGE:
14313                 ret = flow_dv_aso_age_release(dev, idx);
14314                 if (ret)
14315                         /*
14316                          * In this case, the last flow has a reference will
14317                          * actually release the age action.
14318                          */
14319                         DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was"
14320                                 " released with references %d.", idx, ret);
14321                 return 0;
14322         case MLX5_INDIRECT_ACTION_TYPE_CT:
14323                 ret = flow_dv_aso_ct_release(dev, idx);
14324                 if (ret < 0)
14325                         return ret;
14326                 if (ret > 0)
14327                         DRV_LOG(DEBUG, "Connection tracking object %u still "
14328                                 "has references %d.", idx, ret);
14329                 return 0;
14330         default:
14331                 return rte_flow_error_set(error, ENOTSUP,
14332                                           RTE_FLOW_ERROR_TYPE_ACTION,
14333                                           NULL,
14334                                           "action type not supported");
14335         }
14336 }
14337
14338 /**
14339  * Updates in place shared RSS action configuration.
14340  *
14341  * @param[in] dev
14342  *   Pointer to the Ethernet device structure.
14343  * @param[in] idx
14344  *   The shared RSS action object ID to be updated.
14345  * @param[in] action_conf
14346  *   RSS action specification used to modify *shared_rss*.
14347  * @param[out] error
14348  *   Perform verbose error reporting if not NULL. Initialized in case of
14349  *   error only.
14350  *
14351  * @return
14352  *   0 on success, otherwise negative errno value.
14353  * @note: currently only support update of RSS queues.
14354  */
14355 static int
14356 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx,
14357                             const struct rte_flow_action_rss *action_conf,
14358                             struct rte_flow_error *error)
14359 {
14360         struct mlx5_priv *priv = dev->data->dev_private;
14361         struct mlx5_shared_action_rss *shared_rss =
14362             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14363         int ret = 0;
14364         void *queue = NULL;
14365         uint16_t *queue_old = NULL;
14366         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
14367
14368         if (!shared_rss)
14369                 return rte_flow_error_set(error, EINVAL,
14370                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14371                                           "invalid shared action to update");
14372         if (priv->obj_ops.ind_table_modify == NULL)
14373                 return rte_flow_error_set(error, ENOTSUP,
14374                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14375                                           "cannot modify indirection table");
14376         queue = mlx5_malloc(MLX5_MEM_ZERO,
14377                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14378                             0, SOCKET_ID_ANY);
14379         if (!queue)
14380                 return rte_flow_error_set(error, ENOMEM,
14381                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14382                                           NULL,
14383                                           "cannot allocate resource memory");
14384         memcpy(queue, action_conf->queue, queue_size);
14385         MLX5_ASSERT(shared_rss->ind_tbl);
14386         rte_spinlock_lock(&shared_rss->action_rss_sl);
14387         queue_old = shared_rss->ind_tbl->queues;
14388         ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl,
14389                                         queue, action_conf->queue_num, true);
14390         if (ret) {
14391                 mlx5_free(queue);
14392                 ret = rte_flow_error_set(error, rte_errno,
14393                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14394                                           "cannot update indirection table");
14395         } else {
14396                 mlx5_free(queue_old);
14397                 shared_rss->origin.queue = queue;
14398                 shared_rss->origin.queue_num = action_conf->queue_num;
14399         }
14400         rte_spinlock_unlock(&shared_rss->action_rss_sl);
14401         return ret;
14402 }
14403
14404 /*
14405  * Updates in place conntrack context or direction.
14406  * Context update should be synchronized.
14407  *
14408  * @param[in] dev
14409  *   Pointer to the Ethernet device structure.
14410  * @param[in] idx
14411  *   The conntrack object ID to be updated.
14412  * @param[in] update
14413  *   Pointer to the structure of information to update.
14414  * @param[out] error
14415  *   Perform verbose error reporting if not NULL. Initialized in case of
14416  *   error only.
14417  *
14418  * @return
14419  *   0 on success, otherwise negative errno value.
14420  */
14421 static int
14422 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx,
14423                            const struct rte_flow_modify_conntrack *update,
14424                            struct rte_flow_error *error)
14425 {
14426         struct mlx5_priv *priv = dev->data->dev_private;
14427         struct mlx5_aso_ct_action *ct;
14428         const struct rte_flow_action_conntrack *new_prf;
14429         int ret = 0;
14430         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
14431         uint32_t dev_idx;
14432
14433         if (PORT_ID(priv) != owner)
14434                 return rte_flow_error_set(error, EACCES,
14435                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14436                                           NULL,
14437                                           "CT object owned by another port");
14438         dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
14439         ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
14440         if (!ct->refcnt)
14441                 return rte_flow_error_set(error, ENOMEM,
14442                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14443                                           NULL,
14444                                           "CT object is inactive");
14445         new_prf = &update->new_ct;
14446         if (update->direction)
14447                 ct->is_original = !!new_prf->is_original_dir;
14448         if (update->state) {
14449                 /* Only validate the profile when it needs to be updated. */
14450                 ret = mlx5_validate_action_ct(dev, new_prf, error);
14451                 if (ret)
14452                         return ret;
14453                 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf);
14454                 if (ret)
14455                         return rte_flow_error_set(error, EIO,
14456                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14457                                         NULL,
14458                                         "Failed to send CT context update WQE");
14459                 /* Block until ready or a failure. */
14460                 ret = mlx5_aso_ct_available(priv->sh, ct);
14461                 if (ret)
14462                         rte_flow_error_set(error, rte_errno,
14463                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14464                                            NULL,
14465                                            "Timeout to get the CT update");
14466         }
14467         return ret;
14468 }
14469
14470 /**
14471  * Updates in place shared action configuration, lock free,
14472  * (mutex should be acquired by caller).
14473  *
14474  * @param[in] dev
14475  *   Pointer to the Ethernet device structure.
14476  * @param[in] handle
14477  *   The indirect action object handle to be updated.
14478  * @param[in] update
14479  *   Action specification used to modify the action pointed by *handle*.
14480  *   *update* could be of same type with the action pointed by the *handle*
14481  *   handle argument, or some other structures like a wrapper, depending on
14482  *   the indirect action type.
14483  * @param[out] error
14484  *   Perform verbose error reporting if not NULL. Initialized in case of
14485  *   error only.
14486  *
14487  * @return
14488  *   0 on success, otherwise negative errno value.
14489  */
14490 static int
14491 flow_dv_action_update(struct rte_eth_dev *dev,
14492                         struct rte_flow_action_handle *handle,
14493                         const void *update,
14494                         struct rte_flow_error *err)
14495 {
14496         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14497         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14498         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14499         const void *action_conf;
14500
14501         switch (type) {
14502         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14503                 action_conf = ((const struct rte_flow_action *)update)->conf;
14504                 return __flow_dv_action_rss_update(dev, idx, action_conf, err);
14505         case MLX5_INDIRECT_ACTION_TYPE_CT:
14506                 return __flow_dv_action_ct_update(dev, idx, update, err);
14507         default:
14508                 return rte_flow_error_set(err, ENOTSUP,
14509                                           RTE_FLOW_ERROR_TYPE_ACTION,
14510                                           NULL,
14511                                           "action type update not supported");
14512         }
14513 }
14514
14515 /**
14516  * Destroy the meter sub policy table rules.
14517  * Lock free, (mutex should be acquired by caller).
14518  *
14519  * @param[in] dev
14520  *   Pointer to Ethernet device.
14521  * @param[in] sub_policy
14522  *   Pointer to meter sub policy table.
14523  */
14524 static void
14525 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
14526                              struct mlx5_flow_meter_sub_policy *sub_policy)
14527 {
14528         struct mlx5_flow_tbl_data_entry *tbl;
14529         int i;
14530
14531         for (i = 0; i < RTE_COLORS; i++) {
14532                 if (sub_policy->color_rule[i]) {
14533                         claim_zero(mlx5_flow_os_destroy_flow
14534                                 (sub_policy->color_rule[i]));
14535                         sub_policy->color_rule[i] = NULL;
14536                 }
14537                 if (sub_policy->color_matcher[i]) {
14538                         tbl = container_of(sub_policy->color_matcher[i]->tbl,
14539                                 typeof(*tbl), tbl);
14540                         mlx5_cache_unregister(&tbl->matchers,
14541                                       &sub_policy->color_matcher[i]->entry);
14542                         sub_policy->color_matcher[i] = NULL;
14543                 }
14544         }
14545         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
14546                 if (sub_policy->rix_hrxq[i]) {
14547                         mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]);
14548                         sub_policy->rix_hrxq[i] = 0;
14549                 }
14550                 if (sub_policy->jump_tbl[i]) {
14551                         flow_dv_tbl_resource_release(MLX5_SH(dev),
14552                         sub_policy->jump_tbl[i]);
14553                         sub_policy->jump_tbl[i] = NULL;
14554                 }
14555         }
14556         if (sub_policy->tbl_rsc) {
14557                 flow_dv_tbl_resource_release(MLX5_SH(dev),
14558                         sub_policy->tbl_rsc);
14559                 sub_policy->tbl_rsc = NULL;
14560         }
14561 }
14562
14563 /**
14564  * Destroy policy rules, lock free,
14565  * (mutex should be acquired by caller).
14566  * Dispatcher for action type specific call.
14567  *
14568  * @param[in] dev
14569  *   Pointer to the Ethernet device structure.
14570  * @param[in] mtr_policy
14571  *   Meter policy struct.
14572  */
14573 static void
14574 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev,
14575                       struct mlx5_flow_meter_policy *mtr_policy)
14576 {
14577         uint32_t i, j;
14578         struct mlx5_flow_meter_sub_policy *sub_policy;
14579         uint16_t sub_policy_num;
14580
14581         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
14582                 sub_policy_num = (mtr_policy->sub_policy_num >>
14583                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
14584                         MLX5_MTR_SUB_POLICY_NUM_MASK;
14585                 for (j = 0; j < sub_policy_num; j++) {
14586                         sub_policy = mtr_policy->sub_policys[i][j];
14587                         if (sub_policy)
14588                                 __flow_dv_destroy_sub_policy_rules
14589                                                 (dev, sub_policy);
14590                 }
14591         }
14592 }
14593
14594 /**
14595  * Destroy policy action, lock free,
14596  * (mutex should be acquired by caller).
14597  * Dispatcher for action type specific call.
14598  *
14599  * @param[in] dev
14600  *   Pointer to the Ethernet device structure.
14601  * @param[in] mtr_policy
14602  *   Meter policy struct.
14603  */
14604 static void
14605 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
14606                       struct mlx5_flow_meter_policy *mtr_policy)
14607 {
14608         struct rte_flow_action *rss_action;
14609         struct mlx5_flow_handle dev_handle;
14610         uint32_t i, j;
14611
14612         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
14613                 if (mtr_policy->act_cnt[i].rix_mark) {
14614                         flow_dv_tag_release(dev,
14615                                 mtr_policy->act_cnt[i].rix_mark);
14616                         mtr_policy->act_cnt[i].rix_mark = 0;
14617                 }
14618                 if (mtr_policy->act_cnt[i].modify_hdr) {
14619                         dev_handle.dvh.modify_hdr =
14620                                 mtr_policy->act_cnt[i].modify_hdr;
14621                         flow_dv_modify_hdr_resource_release(dev, &dev_handle);
14622                 }
14623                 switch (mtr_policy->act_cnt[i].fate_action) {
14624                 case MLX5_FLOW_FATE_SHARED_RSS:
14625                         rss_action = mtr_policy->act_cnt[i].rss;
14626                         mlx5_free(rss_action);
14627                         break;
14628                 case MLX5_FLOW_FATE_PORT_ID:
14629                         if (mtr_policy->act_cnt[i].rix_port_id_action) {
14630                                 flow_dv_port_id_action_resource_release(dev,
14631                                 mtr_policy->act_cnt[i].rix_port_id_action);
14632                                 mtr_policy->act_cnt[i].rix_port_id_action = 0;
14633                         }
14634                         break;
14635                 case MLX5_FLOW_FATE_DROP:
14636                 case MLX5_FLOW_FATE_JUMP:
14637                         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
14638                                 mtr_policy->act_cnt[i].dr_jump_action[j] =
14639                                                 NULL;
14640                         break;
14641                 default:
14642                         /*Queue action do nothing*/
14643                         break;
14644                 }
14645         }
14646         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
14647                 mtr_policy->dr_drop_action[j] = NULL;
14648 }
14649
14650 /**
14651  * Create policy action per domain, lock free,
14652  * (mutex should be acquired by caller).
14653  * Dispatcher for action type specific call.
14654  *
14655  * @param[in] dev
14656  *   Pointer to the Ethernet device structure.
14657  * @param[in] mtr_policy
14658  *   Meter policy struct.
14659  * @param[in] action
14660  *   Action specification used to create meter actions.
14661  * @param[out] error
14662  *   Perform verbose error reporting if not NULL. Initialized in case of
14663  *   error only.
14664  *
14665  * @return
14666  *   0 on success, otherwise negative errno value.
14667  */
14668 static int
14669 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
14670                         struct mlx5_flow_meter_policy *mtr_policy,
14671                         const struct rte_flow_action *actions[RTE_COLORS],
14672                         enum mlx5_meter_domain domain,
14673                         struct rte_mtr_error *error)
14674 {
14675         struct mlx5_priv *priv = dev->data->dev_private;
14676         struct rte_flow_error flow_err;
14677         const struct rte_flow_action *act;
14678         uint64_t action_flags = 0;
14679         struct mlx5_flow_handle dh;
14680         struct mlx5_flow dev_flow;
14681         struct mlx5_flow_dv_port_id_action_resource port_id_action;
14682         int i, ret;
14683         uint8_t egress, transfer;
14684         struct mlx5_meter_policy_action_container *act_cnt = NULL;
14685         union {
14686                 struct mlx5_flow_dv_modify_hdr_resource res;
14687                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
14688                             sizeof(struct mlx5_modification_cmd) *
14689                             (MLX5_MAX_MODIFY_NUM + 1)];
14690         } mhdr_dummy;
14691
14692         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
14693         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
14694         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
14695         memset(&dev_flow, 0, sizeof(struct mlx5_flow));
14696         memset(&port_id_action, 0,
14697                 sizeof(struct mlx5_flow_dv_port_id_action_resource));
14698         dev_flow.handle = &dh;
14699         dev_flow.dv.port_id_action = &port_id_action;
14700         dev_flow.external = true;
14701         for (i = 0; i < RTE_COLORS; i++) {
14702                 if (i < MLX5_MTR_RTE_COLORS)
14703                         act_cnt = &mtr_policy->act_cnt[i];
14704                 for (act = actions[i];
14705                         act && act->type != RTE_FLOW_ACTION_TYPE_END;
14706                         act++) {
14707                         switch (act->type) {
14708                         case RTE_FLOW_ACTION_TYPE_MARK:
14709                         {
14710                                 uint32_t tag_be = mlx5_flow_mark_set
14711                                         (((const struct rte_flow_action_mark *)
14712                                         (act->conf))->id);
14713
14714                                 if (i >= MLX5_MTR_RTE_COLORS)
14715                                         return -rte_mtr_error_set(error,
14716                                           ENOTSUP,
14717                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14718                                           NULL,
14719                                           "cannot create policy "
14720                                           "mark action for this color");
14721                                 dev_flow.handle->mark = 1;
14722                                 if (flow_dv_tag_resource_register(dev, tag_be,
14723                                                   &dev_flow, &flow_err))
14724                                         return -rte_mtr_error_set(error,
14725                                         ENOTSUP,
14726                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14727                                         NULL,
14728                                         "cannot setup policy mark action");
14729                                 MLX5_ASSERT(dev_flow.dv.tag_resource);
14730                                 act_cnt->rix_mark =
14731                                         dev_flow.handle->dvh.rix_tag;
14732                                 action_flags |= MLX5_FLOW_ACTION_MARK;
14733                                 break;
14734                         }
14735                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
14736                         {
14737                                 struct mlx5_flow_dv_modify_hdr_resource
14738                                         *mhdr_res = &mhdr_dummy.res;
14739
14740                                 if (i >= MLX5_MTR_RTE_COLORS)
14741                                         return -rte_mtr_error_set(error,
14742                                           ENOTSUP,
14743                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14744                                           NULL,
14745                                           "cannot create policy "
14746                                           "set tag action for this color");
14747                                 memset(mhdr_res, 0, sizeof(*mhdr_res));
14748                                 mhdr_res->ft_type = transfer ?
14749                                         MLX5DV_FLOW_TABLE_TYPE_FDB :
14750                                         egress ?
14751                                         MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
14752                                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
14753                                 if (flow_dv_convert_action_set_tag
14754                                 (dev, mhdr_res,
14755                                 (const struct rte_flow_action_set_tag *)
14756                                 act->conf,  &flow_err))
14757                                         return -rte_mtr_error_set(error,
14758                                         ENOTSUP,
14759                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14760                                         NULL, "cannot convert policy "
14761                                         "set tag action");
14762                                 if (!mhdr_res->actions_num)
14763                                         return -rte_mtr_error_set(error,
14764                                         ENOTSUP,
14765                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14766                                         NULL, "cannot find policy "
14767                                         "set tag action");
14768                                 /* create modify action if needed. */
14769                                 dev_flow.dv.group = 1;
14770                                 if (flow_dv_modify_hdr_resource_register
14771                                         (dev, mhdr_res, &dev_flow, &flow_err))
14772                                         return -rte_mtr_error_set(error,
14773                                         ENOTSUP,
14774                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14775                                         NULL, "cannot register policy "
14776                                         "set tag action");
14777                                 act_cnt->modify_hdr =
14778                                 dev_flow.handle->dvh.modify_hdr;
14779                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
14780                                 break;
14781                         }
14782                         case RTE_FLOW_ACTION_TYPE_DROP:
14783                         {
14784                                 struct mlx5_flow_mtr_mng *mtrmng =
14785                                                 priv->sh->mtrmng;
14786                                 struct mlx5_flow_tbl_data_entry *tbl_data;
14787
14788                                 /*
14789                                  * Create the drop table with
14790                                  * METER DROP level.
14791                                  */
14792                                 if (!mtrmng->drop_tbl[domain]) {
14793                                         mtrmng->drop_tbl[domain] =
14794                                         flow_dv_tbl_resource_get(dev,
14795                                         MLX5_FLOW_TABLE_LEVEL_METER,
14796                                         egress, transfer, false, NULL, 0,
14797                                         0, MLX5_MTR_TABLE_ID_DROP, &flow_err);
14798                                         if (!mtrmng->drop_tbl[domain])
14799                                                 return -rte_mtr_error_set
14800                                         (error, ENOTSUP,
14801                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14802                                         NULL,
14803                                         "Failed to create meter drop table");
14804                                 }
14805                                 tbl_data = container_of
14806                                 (mtrmng->drop_tbl[domain],
14807                                 struct mlx5_flow_tbl_data_entry, tbl);
14808                                 if (i < MLX5_MTR_RTE_COLORS) {
14809                                         act_cnt->dr_jump_action[domain] =
14810                                                 tbl_data->jump.action;
14811                                         act_cnt->fate_action =
14812                                                 MLX5_FLOW_FATE_DROP;
14813                                 }
14814                                 if (i == RTE_COLOR_RED)
14815                                         mtr_policy->dr_drop_action[domain] =
14816                                                 tbl_data->jump.action;
14817                                 action_flags |= MLX5_FLOW_ACTION_DROP;
14818                                 break;
14819                         }
14820                         case RTE_FLOW_ACTION_TYPE_QUEUE:
14821                         {
14822                                 if (i >= MLX5_MTR_RTE_COLORS)
14823                                         return -rte_mtr_error_set(error,
14824                                         ENOTSUP,
14825                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14826                                         NULL, "cannot create policy "
14827                                         "fate queue for this color");
14828                                 act_cnt->queue =
14829                                 ((const struct rte_flow_action_queue *)
14830                                         (act->conf))->index;
14831                                 act_cnt->fate_action =
14832                                         MLX5_FLOW_FATE_QUEUE;
14833                                 dev_flow.handle->fate_action =
14834                                         MLX5_FLOW_FATE_QUEUE;
14835                                 mtr_policy->is_queue = 1;
14836                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
14837                                 break;
14838                         }
14839                         case RTE_FLOW_ACTION_TYPE_RSS:
14840                         {
14841                                 int rss_size;
14842
14843                                 if (i >= MLX5_MTR_RTE_COLORS)
14844                                         return -rte_mtr_error_set(error,
14845                                           ENOTSUP,
14846                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14847                                           NULL,
14848                                           "cannot create policy "
14849                                           "rss action for this color");
14850                                 /*
14851                                  * Save RSS conf into policy struct
14852                                  * for translate stage.
14853                                  */
14854                                 rss_size = (int)rte_flow_conv
14855                                         (RTE_FLOW_CONV_OP_ACTION,
14856                                         NULL, 0, act, &flow_err);
14857                                 if (rss_size <= 0)
14858                                         return -rte_mtr_error_set(error,
14859                                           ENOTSUP,
14860                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14861                                           NULL, "Get the wrong "
14862                                           "rss action struct size");
14863                                 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO,
14864                                                 rss_size, 0, SOCKET_ID_ANY);
14865                                 if (!act_cnt->rss)
14866                                         return -rte_mtr_error_set(error,
14867                                           ENOTSUP,
14868                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14869                                           NULL,
14870                                           "Fail to malloc rss action memory");
14871                                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION,
14872                                         act_cnt->rss, rss_size,
14873                                         act, &flow_err);
14874                                 if (ret < 0)
14875                                         return -rte_mtr_error_set(error,
14876                                           ENOTSUP,
14877                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14878                                           NULL, "Fail to save "
14879                                           "rss action into policy struct");
14880                                 act_cnt->fate_action =
14881                                         MLX5_FLOW_FATE_SHARED_RSS;
14882                                 action_flags |= MLX5_FLOW_ACTION_RSS;
14883                                 break;
14884                         }
14885                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
14886                         {
14887                                 struct mlx5_flow_dv_port_id_action_resource
14888                                         port_id_resource;
14889                                 uint32_t port_id = 0;
14890
14891                                 if (i >= MLX5_MTR_RTE_COLORS)
14892                                         return -rte_mtr_error_set(error,
14893                                         ENOTSUP,
14894                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14895                                         NULL, "cannot create policy "
14896                                         "port action for this color");
14897                                 memset(&port_id_resource, 0,
14898                                         sizeof(port_id_resource));
14899                                 if (flow_dv_translate_action_port_id(dev, act,
14900                                                 &port_id, &flow_err))
14901                                         return -rte_mtr_error_set(error,
14902                                         ENOTSUP,
14903                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14904                                         NULL, "cannot translate "
14905                                         "policy port action");
14906                                 port_id_resource.port_id = port_id;
14907                                 if (flow_dv_port_id_action_resource_register
14908                                         (dev, &port_id_resource,
14909                                         &dev_flow, &flow_err))
14910                                         return -rte_mtr_error_set(error,
14911                                         ENOTSUP,
14912                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14913                                         NULL, "cannot setup "
14914                                         "policy port action");
14915                                 act_cnt->rix_port_id_action =
14916                                         dev_flow.handle->rix_port_id_action;
14917                                 act_cnt->fate_action =
14918                                         MLX5_FLOW_FATE_PORT_ID;
14919                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
14920                                 break;
14921                         }
14922                         case RTE_FLOW_ACTION_TYPE_JUMP:
14923                         {
14924                                 uint32_t jump_group = 0;
14925                                 uint32_t table = 0;
14926                                 struct mlx5_flow_tbl_data_entry *tbl_data;
14927                                 struct flow_grp_info grp_info = {
14928                                         .external = !!dev_flow.external,
14929                                         .transfer = !!transfer,
14930                                         .fdb_def_rule = !!priv->fdb_def_rule,
14931                                         .std_tbl_fix = 0,
14932                                         .skip_scale = dev_flow.skip_scale &
14933                                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
14934                                 };
14935                                 struct mlx5_flow_meter_sub_policy *sub_policy =
14936                                 mtr_policy->sub_policys[domain][0];
14937
14938                                 if (i >= MLX5_MTR_RTE_COLORS)
14939                                         return -rte_mtr_error_set(error,
14940                                           ENOTSUP,
14941                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14942                                           NULL,
14943                                           "cannot create policy "
14944                                           "jump action for this color");
14945                                 jump_group =
14946                                 ((const struct rte_flow_action_jump *)
14947                                                         act->conf)->group;
14948                                 if (mlx5_flow_group_to_table(dev, NULL,
14949                                                        jump_group,
14950                                                        &table,
14951                                                        &grp_info, &flow_err))
14952                                         return -rte_mtr_error_set(error,
14953                                         ENOTSUP,
14954                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14955                                         NULL, "cannot setup "
14956                                         "policy jump action");
14957                                 sub_policy->jump_tbl[i] =
14958                                 flow_dv_tbl_resource_get(dev,
14959                                         table, egress,
14960                                         transfer,
14961                                         !!dev_flow.external,
14962                                         NULL, jump_group, 0,
14963                                         0, &flow_err);
14964                                 if
14965                                 (!sub_policy->jump_tbl[i])
14966                                         return  -rte_mtr_error_set(error,
14967                                         ENOTSUP,
14968                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14969                                         NULL, "cannot create jump action.");
14970                                 tbl_data = container_of
14971                                 (sub_policy->jump_tbl[i],
14972                                 struct mlx5_flow_tbl_data_entry, tbl);
14973                                 act_cnt->dr_jump_action[domain] =
14974                                         tbl_data->jump.action;
14975                                 act_cnt->fate_action =
14976                                         MLX5_FLOW_FATE_JUMP;
14977                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
14978                                 break;
14979                         }
14980                         default:
14981                                 return -rte_mtr_error_set(error, ENOTSUP,
14982                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14983                                           NULL, "action type not supported");
14984                         }
14985                 }
14986         }
14987         return 0;
14988 }
14989
14990 /**
14991  * Create policy action per domain, lock free,
14992  * (mutex should be acquired by caller).
14993  * Dispatcher for action type specific call.
14994  *
14995  * @param[in] dev
14996  *   Pointer to the Ethernet device structure.
14997  * @param[in] mtr_policy
14998  *   Meter policy struct.
14999  * @param[in] action
15000  *   Action specification used to create meter actions.
15001  * @param[out] error
15002  *   Perform verbose error reporting if not NULL. Initialized in case of
15003  *   error only.
15004  *
15005  * @return
15006  *   0 on success, otherwise negative errno value.
15007  */
15008 static int
15009 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev,
15010                       struct mlx5_flow_meter_policy *mtr_policy,
15011                       const struct rte_flow_action *actions[RTE_COLORS],
15012                       struct rte_mtr_error *error)
15013 {
15014         int ret, i;
15015         uint16_t sub_policy_num;
15016
15017         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15018                 sub_policy_num = (mtr_policy->sub_policy_num >>
15019                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15020                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15021                 if (sub_policy_num) {
15022                         ret = __flow_dv_create_domain_policy_acts(dev,
15023                                 mtr_policy, actions,
15024                                 (enum mlx5_meter_domain)i, error);
15025                         if (ret)
15026                                 return ret;
15027                 }
15028         }
15029         return 0;
15030 }
15031
15032 /**
15033  * Query a DV flow rule for its statistics via DevX.
15034  *
15035  * @param[in] dev
15036  *   Pointer to Ethernet device.
15037  * @param[in] cnt_idx
15038  *   Index to the flow counter.
15039  * @param[out] data
15040  *   Data retrieved by the query.
15041  * @param[out] error
15042  *   Perform verbose error reporting if not NULL.
15043  *
15044  * @return
15045  *   0 on success, a negative errno value otherwise and rte_errno is set.
15046  */
15047 static int
15048 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data,
15049                     struct rte_flow_error *error)
15050 {
15051         struct mlx5_priv *priv = dev->data->dev_private;
15052         struct rte_flow_query_count *qc = data;
15053
15054         if (!priv->config.devx)
15055                 return rte_flow_error_set(error, ENOTSUP,
15056                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15057                                           NULL,
15058                                           "counters are not supported");
15059         if (cnt_idx) {
15060                 uint64_t pkts, bytes;
15061                 struct mlx5_flow_counter *cnt;
15062                 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes);
15063
15064                 if (err)
15065                         return rte_flow_error_set(error, -err,
15066                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15067                                         NULL, "cannot read counters");
15068                 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL);
15069                 qc->hits_set = 1;
15070                 qc->bytes_set = 1;
15071                 qc->hits = pkts - cnt->hits;
15072                 qc->bytes = bytes - cnt->bytes;
15073                 if (qc->reset) {
15074                         cnt->hits = pkts;
15075                         cnt->bytes = bytes;
15076                 }
15077                 return 0;
15078         }
15079         return rte_flow_error_set(error, EINVAL,
15080                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15081                                   NULL,
15082                                   "counters are not available");
15083 }
15084
15085 static int
15086 flow_dv_action_query(struct rte_eth_dev *dev,
15087                      const struct rte_flow_action_handle *handle, void *data,
15088                      struct rte_flow_error *error)
15089 {
15090         struct mlx5_age_param *age_param;
15091         struct rte_flow_query_age *resp;
15092         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15093         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15094         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15095         struct mlx5_priv *priv = dev->data->dev_private;
15096         struct mlx5_aso_ct_action *ct;
15097         uint16_t owner;
15098         uint32_t dev_idx;
15099
15100         switch (type) {
15101         case MLX5_INDIRECT_ACTION_TYPE_AGE:
15102                 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params;
15103                 resp = data;
15104                 resp->aged = __atomic_load_n(&age_param->state,
15105                                               __ATOMIC_RELAXED) == AGE_TMOUT ?
15106                                                                           1 : 0;
15107                 resp->sec_since_last_hit_valid = !resp->aged;
15108                 if (resp->sec_since_last_hit_valid)
15109                         resp->sec_since_last_hit = __atomic_load_n
15110                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15111                 return 0;
15112         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
15113                 return flow_dv_query_count(dev, idx, data, error);
15114         case MLX5_INDIRECT_ACTION_TYPE_CT:
15115                 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15116                 if (owner != PORT_ID(priv))
15117                         return rte_flow_error_set(error, EACCES,
15118                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15119                                         NULL,
15120                                         "CT object owned by another port");
15121                 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15122                 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15123                 MLX5_ASSERT(ct);
15124                 if (!ct->refcnt)
15125                         return rte_flow_error_set(error, EFAULT,
15126                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15127                                         NULL,
15128                                         "CT object is inactive");
15129                 ((struct rte_flow_action_conntrack *)data)->peer_port =
15130                                                         ct->peer;
15131                 ((struct rte_flow_action_conntrack *)data)->is_original_dir =
15132                                                         ct->is_original;
15133                 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data))
15134                         return rte_flow_error_set(error, EIO,
15135                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15136                                         NULL,
15137                                         "Failed to query CT context");
15138                 return 0;
15139         default:
15140                 return rte_flow_error_set(error, ENOTSUP,
15141                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15142                                           "action type query not supported");
15143         }
15144 }
15145
15146 /**
15147  * Query a flow rule AGE action for aging information.
15148  *
15149  * @param[in] dev
15150  *   Pointer to Ethernet device.
15151  * @param[in] flow
15152  *   Pointer to the sub flow.
15153  * @param[out] data
15154  *   data retrieved by the query.
15155  * @param[out] error
15156  *   Perform verbose error reporting if not NULL.
15157  *
15158  * @return
15159  *   0 on success, a negative errno value otherwise and rte_errno is set.
15160  */
15161 static int
15162 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
15163                   void *data, struct rte_flow_error *error)
15164 {
15165         struct rte_flow_query_age *resp = data;
15166         struct mlx5_age_param *age_param;
15167
15168         if (flow->age) {
15169                 struct mlx5_aso_age_action *act =
15170                                      flow_aso_age_get_by_idx(dev, flow->age);
15171
15172                 age_param = &act->age_params;
15173         } else if (flow->counter) {
15174                 age_param = flow_dv_counter_idx_get_age(dev, flow->counter);
15175
15176                 if (!age_param || !age_param->timeout)
15177                         return rte_flow_error_set
15178                                         (error, EINVAL,
15179                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15180                                          NULL, "cannot read age data");
15181         } else {
15182                 return rte_flow_error_set(error, EINVAL,
15183                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15184                                           NULL, "age data not available");
15185         }
15186         resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
15187                                      AGE_TMOUT ? 1 : 0;
15188         resp->sec_since_last_hit_valid = !resp->aged;
15189         if (resp->sec_since_last_hit_valid)
15190                 resp->sec_since_last_hit = __atomic_load_n
15191                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15192         return 0;
15193 }
15194
15195 /**
15196  * Query a flow.
15197  *
15198  * @see rte_flow_query()
15199  * @see rte_flow_ops
15200  */
15201 static int
15202 flow_dv_query(struct rte_eth_dev *dev,
15203               struct rte_flow *flow __rte_unused,
15204               const struct rte_flow_action *actions __rte_unused,
15205               void *data __rte_unused,
15206               struct rte_flow_error *error __rte_unused)
15207 {
15208         int ret = -EINVAL;
15209
15210         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
15211                 switch (actions->type) {
15212                 case RTE_FLOW_ACTION_TYPE_VOID:
15213                         break;
15214                 case RTE_FLOW_ACTION_TYPE_COUNT:
15215                         ret = flow_dv_query_count(dev, flow->counter, data,
15216                                                   error);
15217                         break;
15218                 case RTE_FLOW_ACTION_TYPE_AGE:
15219                         ret = flow_dv_query_age(dev, flow, data, error);
15220                         break;
15221                 default:
15222                         return rte_flow_error_set(error, ENOTSUP,
15223                                                   RTE_FLOW_ERROR_TYPE_ACTION,
15224                                                   actions,
15225                                                   "action not supported");
15226                 }
15227         }
15228         return ret;
15229 }
15230
15231 /**
15232  * Destroy the meter table set.
15233  * Lock free, (mutex should be acquired by caller).
15234  *
15235  * @param[in] dev
15236  *   Pointer to Ethernet device.
15237  * @param[in] fm
15238  *   Meter information table.
15239  */
15240 static void
15241 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
15242                         struct mlx5_flow_meter_info *fm)
15243 {
15244         struct mlx5_priv *priv = dev->data->dev_private;
15245         int i;
15246
15247         if (!fm || !priv->config.dv_flow_en)
15248                 return;
15249         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15250                 if (fm->drop_rule[i]) {
15251                         claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
15252                         fm->drop_rule[i] = NULL;
15253                 }
15254         }
15255 }
15256
15257 static void
15258 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
15259 {
15260         struct mlx5_priv *priv = dev->data->dev_private;
15261         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15262         struct mlx5_flow_tbl_data_entry *tbl;
15263         int i, j;
15264
15265         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15266                 if (mtrmng->def_rule[i]) {
15267                         claim_zero(mlx5_flow_os_destroy_flow
15268                                         (mtrmng->def_rule[i]));
15269                         mtrmng->def_rule[i] = NULL;
15270                 }
15271                 if (mtrmng->def_matcher[i]) {
15272                         tbl = container_of(mtrmng->def_matcher[i]->tbl,
15273                                 struct mlx5_flow_tbl_data_entry, tbl);
15274                         mlx5_cache_unregister(&tbl->matchers,
15275                                       &mtrmng->def_matcher[i]->entry);
15276                         mtrmng->def_matcher[i] = NULL;
15277                 }
15278                 for (j = 0; j < MLX5_REG_BITS; j++) {
15279                         if (mtrmng->drop_matcher[i][j]) {
15280                                 tbl =
15281                                 container_of(mtrmng->drop_matcher[i][j]->tbl,
15282                                              struct mlx5_flow_tbl_data_entry,
15283                                              tbl);
15284                                 mlx5_cache_unregister(&tbl->matchers,
15285                                         &mtrmng->drop_matcher[i][j]->entry);
15286                                 mtrmng->drop_matcher[i][j] = NULL;
15287                         }
15288                 }
15289                 if (mtrmng->drop_tbl[i]) {
15290                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15291                                 mtrmng->drop_tbl[i]);
15292                         mtrmng->drop_tbl[i] = NULL;
15293                 }
15294         }
15295 }
15296
15297 /* Number of meter flow actions, count and jump or count and drop. */
15298 #define METER_ACTIONS 2
15299
15300 static void
15301 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
15302                               enum mlx5_meter_domain domain)
15303 {
15304         struct mlx5_priv *priv = dev->data->dev_private;
15305         struct mlx5_flow_meter_def_policy *def_policy =
15306                         priv->sh->mtrmng->def_policy[domain];
15307
15308         __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
15309         mlx5_free(def_policy);
15310         priv->sh->mtrmng->def_policy[domain] = NULL;
15311 }
15312
15313 /**
15314  * Destroy the default policy table set.
15315  *
15316  * @param[in] dev
15317  *   Pointer to Ethernet device.
15318  */
15319 static void
15320 flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
15321 {
15322         struct mlx5_priv *priv = dev->data->dev_private;
15323         int i;
15324
15325         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
15326                 if (priv->sh->mtrmng->def_policy[i])
15327                         __flow_dv_destroy_domain_def_policy(dev,
15328                                         (enum mlx5_meter_domain)i);
15329         priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
15330 }
15331
15332 static int
15333 __flow_dv_create_policy_flow(struct rte_eth_dev *dev,
15334                         uint32_t color_reg_c_idx,
15335                         enum rte_color color, void *matcher_object,
15336                         int actions_n, void *actions,
15337                         bool is_default_policy, void **rule,
15338                         const struct rte_flow_attr *attr)
15339 {
15340         int ret;
15341         struct mlx5_flow_dv_match_params value = {
15342                 .size = sizeof(value.buf) -
15343                         MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15344         };
15345         struct mlx5_flow_dv_match_params matcher = {
15346                 .size = sizeof(matcher.buf) -
15347                         MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15348         };
15349         struct mlx5_priv *priv = dev->data->dev_private;
15350
15351         if (!is_default_policy && (priv->representor || priv->master)) {
15352                 if (flow_dv_translate_item_port_id(dev, matcher.buf,
15353                                                    value.buf, NULL, attr)) {
15354                         DRV_LOG(ERR,
15355                         "Failed to create meter policy flow with port.");
15356                         return -1;
15357                 }
15358         }
15359         flow_dv_match_meta_reg(matcher.buf, value.buf,
15360                                 (enum modify_reg)color_reg_c_idx,
15361                                 rte_col_2_mlx5_col(color),
15362                                 UINT32_MAX);
15363         ret = mlx5_flow_os_create_flow(matcher_object,
15364                         (void *)&value, actions_n, actions, rule);
15365         if (ret) {
15366                 DRV_LOG(ERR, "Failed to create meter policy flow.");
15367                 return -1;
15368         }
15369         return 0;
15370 }
15371
15372 static int
15373 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
15374                         uint32_t color_reg_c_idx,
15375                         uint16_t priority,
15376                         struct mlx5_flow_meter_sub_policy *sub_policy,
15377                         const struct rte_flow_attr *attr,
15378                         bool is_default_policy,
15379                         struct rte_flow_error *error)
15380 {
15381         struct mlx5_cache_entry *entry;
15382         struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
15383         struct mlx5_flow_dv_matcher matcher = {
15384                 .mask = {
15385                         .size = sizeof(matcher.mask.buf) -
15386                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15387                 },
15388                 .tbl = tbl_rsc,
15389         };
15390         struct mlx5_flow_dv_match_params value = {
15391                 .size = sizeof(value.buf) -
15392                         MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15393         };
15394         struct mlx5_flow_cb_ctx ctx = {
15395                 .error = error,
15396                 .data = &matcher,
15397         };
15398         struct mlx5_flow_tbl_data_entry *tbl_data;
15399         struct mlx5_priv *priv = dev->data->dev_private;
15400         uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
15401
15402         if (!is_default_policy && (priv->representor || priv->master)) {
15403                 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
15404                                                    value.buf, NULL, attr)) {
15405                         DRV_LOG(ERR,
15406                         "Failed to register meter drop matcher with port.");
15407                         return -1;
15408                 }
15409         }
15410         tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
15411         if (priority < RTE_COLOR_RED)
15412                 flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
15413                         (enum modify_reg)color_reg_c_idx, 0, color_mask);
15414         matcher.priority = priority;
15415         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
15416                                         matcher.mask.size);
15417         entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
15418         if (!entry) {
15419                 DRV_LOG(ERR, "Failed to register meter drop matcher.");
15420                 return -1;
15421         }
15422         sub_policy->color_matcher[priority] =
15423                 container_of(entry, struct mlx5_flow_dv_matcher, entry);
15424         return 0;
15425 }
15426
15427 /**
15428  * Create the policy rules per domain.
15429  *
15430  * @param[in] dev
15431  *   Pointer to Ethernet device.
15432  * @param[in] sub_policy
15433  *    Pointer to sub policy table..
15434  * @param[in] egress
15435  *   Direction of the table.
15436  * @param[in] transfer
15437  *   E-Switch or NIC flow.
15438  * @param[in] acts
15439  *   Pointer to policy action list per color.
15440  *
15441  * @return
15442  *   0 on success, -1 otherwise.
15443  */
15444 static int
15445 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
15446                 struct mlx5_flow_meter_sub_policy *sub_policy,
15447                 uint8_t egress, uint8_t transfer, bool is_default_policy,
15448                 struct mlx5_meter_policy_acts acts[RTE_COLORS])
15449 {
15450         struct rte_flow_error flow_err;
15451         uint32_t color_reg_c_idx;
15452         struct rte_flow_attr attr = {
15453                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
15454                 .priority = 0,
15455                 .ingress = 0,
15456                 .egress = !!egress,
15457                 .transfer = !!transfer,
15458                 .reserved = 0,
15459         };
15460         int i;
15461         int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
15462
15463         if (ret < 0)
15464                 return -1;
15465         /* Create policy table with POLICY level. */
15466         if (!sub_policy->tbl_rsc)
15467                 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
15468                                 MLX5_FLOW_TABLE_LEVEL_POLICY,
15469                                 egress, transfer, false, NULL, 0, 0,
15470                                 sub_policy->idx, &flow_err);
15471         if (!sub_policy->tbl_rsc) {
15472                 DRV_LOG(ERR,
15473                         "Failed to create meter sub policy table.");
15474                 return -1;
15475         }
15476         /* Prepare matchers. */
15477         color_reg_c_idx = ret;
15478         for (i = 0; i < RTE_COLORS; i++) {
15479                 if (i == RTE_COLOR_YELLOW || !acts[i].actions_n)
15480                         continue;
15481                 attr.priority = i;
15482                 if (!sub_policy->color_matcher[i]) {
15483                         /* Create matchers for Color. */
15484                         if (__flow_dv_create_policy_matcher(dev,
15485                                 color_reg_c_idx, i, sub_policy,
15486                                 &attr, is_default_policy, &flow_err))
15487                                 return -1;
15488                 }
15489                 /* Create flow, matching color. */
15490                 if (acts[i].actions_n)
15491                         if (__flow_dv_create_policy_flow(dev,
15492                                 color_reg_c_idx, (enum rte_color)i,
15493                                 sub_policy->color_matcher[i]->matcher_object,
15494                                 acts[i].actions_n,
15495                                 acts[i].dv_actions,
15496                                 is_default_policy,
15497                                 &sub_policy->color_rule[i],
15498                                 &attr))
15499                                 return -1;
15500         }
15501         return 0;
15502 }
15503
15504 static int
15505 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
15506                         struct mlx5_flow_meter_policy *mtr_policy,
15507                         struct mlx5_flow_meter_sub_policy *sub_policy,
15508                         uint32_t domain)
15509 {
15510         struct mlx5_priv *priv = dev->data->dev_private;
15511         struct mlx5_meter_policy_acts acts[RTE_COLORS];
15512         struct mlx5_flow_dv_tag_resource *tag;
15513         struct mlx5_flow_dv_port_id_action_resource *port_action;
15514         struct mlx5_hrxq *hrxq;
15515         uint8_t egress, transfer;
15516         int i;
15517
15518         for (i = 0; i < RTE_COLORS; i++) {
15519                 acts[i].actions_n = 0;
15520                 if (i == RTE_COLOR_YELLOW)
15521                         continue;
15522                 if (i == RTE_COLOR_RED) {
15523                         /* Only support drop on red. */
15524                         acts[i].dv_actions[0] =
15525                         mtr_policy->dr_drop_action[domain];
15526                         acts[i].actions_n = 1;
15527                         continue;
15528                 }
15529                 if (mtr_policy->act_cnt[i].rix_mark) {
15530                         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
15531                                         mtr_policy->act_cnt[i].rix_mark);
15532                         if (!tag) {
15533                                 DRV_LOG(ERR, "Failed to find "
15534                                 "mark action for policy.");
15535                                 return -1;
15536                         }
15537                         acts[i].dv_actions[acts[i].actions_n] =
15538                                                 tag->action;
15539                         acts[i].actions_n++;
15540                 }
15541                 if (mtr_policy->act_cnt[i].modify_hdr) {
15542                         acts[i].dv_actions[acts[i].actions_n] =
15543                         mtr_policy->act_cnt[i].modify_hdr->action;
15544                         acts[i].actions_n++;
15545                 }
15546                 if (mtr_policy->act_cnt[i].fate_action) {
15547                         switch (mtr_policy->act_cnt[i].fate_action) {
15548                         case MLX5_FLOW_FATE_PORT_ID:
15549                                 port_action = mlx5_ipool_get
15550                                         (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
15551                                 mtr_policy->act_cnt[i].rix_port_id_action);
15552                                 if (!port_action) {
15553                                         DRV_LOG(ERR, "Failed to find "
15554                                                 "port action for policy.");
15555                                         return -1;
15556                                 }
15557                                 acts[i].dv_actions[acts[i].actions_n] =
15558                                 port_action->action;
15559                                 acts[i].actions_n++;
15560                                 break;
15561                         case MLX5_FLOW_FATE_DROP:
15562                         case MLX5_FLOW_FATE_JUMP:
15563                                 acts[i].dv_actions[acts[i].actions_n] =
15564                                 mtr_policy->act_cnt[i].dr_jump_action[domain];
15565                                 acts[i].actions_n++;
15566                                 break;
15567                         case MLX5_FLOW_FATE_SHARED_RSS:
15568                         case MLX5_FLOW_FATE_QUEUE:
15569                                 hrxq = mlx5_ipool_get
15570                                 (priv->sh->ipool[MLX5_IPOOL_HRXQ],
15571                                 sub_policy->rix_hrxq[i]);
15572                                 if (!hrxq) {
15573                                         DRV_LOG(ERR, "Failed to find "
15574                                                 "queue action for policy.");
15575                                         return -1;
15576                                 }
15577                                 acts[i].dv_actions[acts[i].actions_n] =
15578                                 hrxq->action;
15579                                 acts[i].actions_n++;
15580                                 break;
15581                         default:
15582                                 /*Queue action do nothing*/
15583                                 break;
15584                         }
15585                 }
15586         }
15587         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15588         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15589         if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
15590                                 egress, transfer, false, acts)) {
15591                 DRV_LOG(ERR,
15592                 "Failed to create policy rules per domain.");
15593                 return -1;
15594         }
15595         return 0;
15596 }
15597
15598 /**
15599  * Create the policy rules.
15600  *
15601  * @param[in] dev
15602  *   Pointer to Ethernet device.
15603  * @param[in,out] mtr_policy
15604  *   Pointer to meter policy table.
15605  *
15606  * @return
15607  *   0 on success, -1 otherwise.
15608  */
15609 static int
15610 flow_dv_create_policy_rules(struct rte_eth_dev *dev,
15611                              struct mlx5_flow_meter_policy *mtr_policy)
15612 {
15613         int i;
15614         uint16_t sub_policy_num;
15615
15616         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15617                 sub_policy_num = (mtr_policy->sub_policy_num >>
15618                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15619                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15620                 if (!sub_policy_num)
15621                         continue;
15622                 /* Prepare actions list and create policy rules. */
15623                 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
15624                         mtr_policy->sub_policys[i][0], i)) {
15625                         DRV_LOG(ERR,
15626                         "Failed to create policy action list per domain.");
15627                         return -1;
15628                 }
15629         }
15630         return 0;
15631 }
15632
15633 static int
15634 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
15635 {
15636         struct mlx5_priv *priv = dev->data->dev_private;
15637         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15638         struct mlx5_flow_meter_def_policy *def_policy;
15639         struct mlx5_flow_tbl_resource *jump_tbl;
15640         struct mlx5_flow_tbl_data_entry *tbl_data;
15641         uint8_t egress, transfer;
15642         struct rte_flow_error error;
15643         struct mlx5_meter_policy_acts acts[RTE_COLORS];
15644         int ret;
15645
15646         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15647         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15648         def_policy = mtrmng->def_policy[domain];
15649         if (!def_policy) {
15650                 def_policy = mlx5_malloc(MLX5_MEM_ZERO,
15651                         sizeof(struct mlx5_flow_meter_def_policy),
15652                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
15653                 if (!def_policy) {
15654                         DRV_LOG(ERR, "Failed to alloc "
15655                                         "default policy table.");
15656                         goto def_policy_error;
15657                 }
15658                 mtrmng->def_policy[domain] = def_policy;
15659                 /* Create the meter suffix table with SUFFIX level. */
15660                 jump_tbl = flow_dv_tbl_resource_get(dev,
15661                                 MLX5_FLOW_TABLE_LEVEL_METER,
15662                                 egress, transfer, false, NULL, 0,
15663                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
15664                 if (!jump_tbl) {
15665                         DRV_LOG(ERR,
15666                                 "Failed to create meter suffix table.");
15667                         goto def_policy_error;
15668                 }
15669                 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
15670                 tbl_data = container_of(jump_tbl,
15671                                 struct mlx5_flow_tbl_data_entry, tbl);
15672                 def_policy->dr_jump_action[RTE_COLOR_GREEN] =
15673                                                 tbl_data->jump.action;
15674                 acts[RTE_COLOR_GREEN].dv_actions[0] =
15675                                                 tbl_data->jump.action;
15676                 acts[RTE_COLOR_GREEN].actions_n = 1;
15677                 /* Create jump action to the drop table. */
15678                 if (!mtrmng->drop_tbl[domain]) {
15679                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
15680                                 (dev, MLX5_FLOW_TABLE_LEVEL_METER,
15681                                 egress, transfer, false, NULL, 0,
15682                                 0, MLX5_MTR_TABLE_ID_DROP, &error);
15683                         if (!mtrmng->drop_tbl[domain]) {
15684                                 DRV_LOG(ERR, "Failed to create "
15685                                 "meter drop table for default policy.");
15686                                 goto def_policy_error;
15687                         }
15688                 }
15689                 tbl_data = container_of(mtrmng->drop_tbl[domain],
15690                                 struct mlx5_flow_tbl_data_entry, tbl);
15691                 def_policy->dr_jump_action[RTE_COLOR_RED] =
15692                                                 tbl_data->jump.action;
15693                 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
15694                 acts[RTE_COLOR_RED].actions_n = 1;
15695                 /* Create default policy rules. */
15696                 ret = __flow_dv_create_domain_policy_rules(dev,
15697                                         &def_policy->sub_policy,
15698                                         egress, transfer, true, acts);
15699                 if (ret) {
15700                         DRV_LOG(ERR, "Failed to create "
15701                                 "default policy rules.");
15702                                 goto def_policy_error;
15703                 }
15704         }
15705         return 0;
15706 def_policy_error:
15707         __flow_dv_destroy_domain_def_policy(dev,
15708                         (enum mlx5_meter_domain)domain);
15709         return -1;
15710 }
15711
15712 /**
15713  * Create the default policy table set.
15714  *
15715  * @param[in] dev
15716  *   Pointer to Ethernet device.
15717  * @return
15718  *   0 on success, -1 otherwise.
15719  */
15720 static int
15721 flow_dv_create_def_policy(struct rte_eth_dev *dev)
15722 {
15723         struct mlx5_priv *priv = dev->data->dev_private;
15724         int i;
15725
15726         /* Non-termination policy table. */
15727         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15728                 if (!priv->config.dv_esw_en && i == MLX5_MTR_DOMAIN_TRANSFER)
15729                         continue;
15730                 if (__flow_dv_create_domain_def_policy(dev, i)) {
15731                         DRV_LOG(ERR,
15732                         "Failed to create default policy");
15733                         return -1;
15734                 }
15735         }
15736         return 0;
15737 }
15738
15739 /**
15740  * Create the needed meter tables.
15741  * Lock free, (mutex should be acquired by caller).
15742  *
15743  * @param[in] dev
15744  *   Pointer to Ethernet device.
15745  * @param[in] fm
15746  *   Meter information table.
15747  * @param[in] mtr_idx
15748  *   Meter index.
15749  * @param[in] domain_bitmap
15750  *   Domain bitmap.
15751  * @return
15752  *   0 on success, -1 otherwise.
15753  */
15754 static int
15755 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
15756                         struct mlx5_flow_meter_info *fm,
15757                         uint32_t mtr_idx,
15758                         uint8_t domain_bitmap)
15759 {
15760         struct mlx5_priv *priv = dev->data->dev_private;
15761         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15762         struct rte_flow_error error;
15763         struct mlx5_flow_tbl_data_entry *tbl_data;
15764         uint8_t egress, transfer;
15765         void *actions[METER_ACTIONS];
15766         int domain, ret, i;
15767         struct mlx5_flow_counter *cnt;
15768         struct mlx5_flow_dv_match_params value = {
15769                 .size = sizeof(value.buf) -
15770                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15771         };
15772         struct mlx5_flow_dv_match_params matcher_para = {
15773                 .size = sizeof(matcher_para.buf) -
15774                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15775         };
15776         int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
15777                                                      0, &error);
15778         uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
15779         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
15780         struct mlx5_cache_entry *entry;
15781         struct mlx5_flow_dv_matcher matcher = {
15782                 .mask = {
15783                         .size = sizeof(matcher.mask.buf) -
15784                         MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15785                 },
15786         };
15787         struct mlx5_flow_dv_matcher *drop_matcher;
15788         struct mlx5_flow_cb_ctx ctx = {
15789                 .error = &error,
15790                 .data = &matcher,
15791         };
15792
15793         if (!priv->mtr_en || mtr_id_reg_c < 0) {
15794                 rte_errno = ENOTSUP;
15795                 return -1;
15796         }
15797         for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
15798                 if (!(domain_bitmap & (1 << domain)) ||
15799                         (mtrmng->def_rule[domain] && !fm->drop_cnt))
15800                         continue;
15801                 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15802                 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15803                 /* Create the drop table with METER DROP level. */
15804                 if (!mtrmng->drop_tbl[domain]) {
15805                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
15806                                         MLX5_FLOW_TABLE_LEVEL_METER,
15807                                         egress, transfer, false, NULL, 0,
15808                                         0, MLX5_MTR_TABLE_ID_DROP, &error);
15809                         if (!mtrmng->drop_tbl[domain]) {
15810                                 DRV_LOG(ERR, "Failed to create meter drop table.");
15811                                 goto policy_error;
15812                         }
15813                 }
15814                 /* Create default matcher in drop table. */
15815                 matcher.tbl = mtrmng->drop_tbl[domain],
15816                 tbl_data = container_of(mtrmng->drop_tbl[domain],
15817                                 struct mlx5_flow_tbl_data_entry, tbl);
15818                 if (!mtrmng->def_matcher[domain]) {
15819                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
15820                                        (enum modify_reg)mtr_id_reg_c,
15821                                        0, 0);
15822                         matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
15823                         matcher.crc = rte_raw_cksum
15824                                         ((const void *)matcher.mask.buf,
15825                                         matcher.mask.size);
15826                         entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
15827                         if (!entry) {
15828                                 DRV_LOG(ERR, "Failed to register meter "
15829                                 "drop default matcher.");
15830                                 goto policy_error;
15831                         }
15832                         mtrmng->def_matcher[domain] = container_of(entry,
15833                         struct mlx5_flow_dv_matcher, entry);
15834                 }
15835                 /* Create default rule in drop table. */
15836                 if (!mtrmng->def_rule[domain]) {
15837                         i = 0;
15838                         actions[i++] = priv->sh->dr_drop_action;
15839                         flow_dv_match_meta_reg(matcher_para.buf, value.buf,
15840                                 (enum modify_reg)mtr_id_reg_c, 0, 0);
15841                         ret = mlx5_flow_os_create_flow
15842                                 (mtrmng->def_matcher[domain]->matcher_object,
15843                                 (void *)&value, i, actions,
15844                                 &mtrmng->def_rule[domain]);
15845                         if (ret) {
15846                                 DRV_LOG(ERR, "Failed to create meter "
15847                                 "default drop rule for drop table.");
15848                                 goto policy_error;
15849                         }
15850                 }
15851                 if (!fm->drop_cnt)
15852                         continue;
15853                 MLX5_ASSERT(mtrmng->max_mtr_bits);
15854                 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
15855                         /* Create matchers for Drop. */
15856                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
15857                                         (enum modify_reg)mtr_id_reg_c, 0,
15858                                         (mtr_id_mask << mtr_id_offset));
15859                         matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
15860                         matcher.crc = rte_raw_cksum
15861                                         ((const void *)matcher.mask.buf,
15862                                         matcher.mask.size);
15863                         entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
15864                         if (!entry) {
15865                                 DRV_LOG(ERR,
15866                                 "Failed to register meter drop matcher.");
15867                                 goto policy_error;
15868                         }
15869                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
15870                                 container_of(entry, struct mlx5_flow_dv_matcher,
15871                                              entry);
15872                 }
15873                 drop_matcher =
15874                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
15875                 /* Create drop rule, matching meter_id only. */
15876                 flow_dv_match_meta_reg(matcher_para.buf, value.buf,
15877                                 (enum modify_reg)mtr_id_reg_c,
15878                                 (mtr_idx << mtr_id_offset), UINT32_MAX);
15879                 i = 0;
15880                 cnt = flow_dv_counter_get_by_idx(dev,
15881                                         fm->drop_cnt, NULL);
15882                 actions[i++] = cnt->action;
15883                 actions[i++] = priv->sh->dr_drop_action;
15884                 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
15885                                                (void *)&value, i, actions,
15886                                                &fm->drop_rule[domain]);
15887                 if (ret) {
15888                         DRV_LOG(ERR, "Failed to create meter "
15889                                 "drop rule for drop table.");
15890                                 goto policy_error;
15891                 }
15892         }
15893         return 0;
15894 policy_error:
15895         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15896                 if (fm->drop_rule[i]) {
15897                         claim_zero(mlx5_flow_os_destroy_flow
15898                                 (fm->drop_rule[i]));
15899                         fm->drop_rule[i] = NULL;
15900                 }
15901         }
15902         return -1;
15903 }
15904
15905 /**
15906  * Find the policy table for prefix table with RSS.
15907  *
15908  * @param[in] dev
15909  *   Pointer to Ethernet device.
15910  * @param[in] mtr_policy
15911  *   Pointer to meter policy table.
15912  * @param[in] rss_desc
15913  *   Pointer to rss_desc
15914  * @return
15915  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
15916  */
15917 static struct mlx5_flow_meter_sub_policy *
15918 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
15919                 struct mlx5_flow_meter_policy *mtr_policy,
15920                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
15921 {
15922         struct mlx5_priv *priv = dev->data->dev_private;
15923         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
15924         uint32_t sub_policy_idx = 0;
15925         uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
15926         uint32_t i, j;
15927         struct mlx5_hrxq *hrxq;
15928         struct mlx5_flow_handle dh;
15929         struct mlx5_meter_policy_action_container *act_cnt;
15930         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
15931         uint16_t sub_policy_num;
15932
15933         rte_spinlock_lock(&mtr_policy->sl);
15934         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15935                 if (!rss_desc[i])
15936                         continue;
15937                 hrxq_idx[i] = mlx5_hrxq_get(dev, rss_desc[i]);
15938                 if (!hrxq_idx[i]) {
15939                         rte_spinlock_unlock(&mtr_policy->sl);
15940                         return NULL;
15941                 }
15942         }
15943         sub_policy_num = (mtr_policy->sub_policy_num >>
15944                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
15945                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15946         for (i = 0; i < sub_policy_num;
15947                 i++) {
15948                 for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) {
15949                         if (rss_desc[j] &&
15950                                 hrxq_idx[j] !=
15951                         mtr_policy->sub_policys[domain][i]->rix_hrxq[j])
15952                                 break;
15953                 }
15954                 if (j >= MLX5_MTR_RTE_COLORS) {
15955                         /*
15956                          * Found the sub policy table with
15957                          * the same queue per color
15958                          */
15959                         rte_spinlock_unlock(&mtr_policy->sl);
15960                         for (j = 0; j < MLX5_MTR_RTE_COLORS; j++)
15961                                 mlx5_hrxq_release(dev, hrxq_idx[j]);
15962                         return mtr_policy->sub_policys[domain][i];
15963                 }
15964         }
15965         /* Create sub policy. */
15966         if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
15967                 /* Reuse the first dummy sub_policy*/
15968                 sub_policy = mtr_policy->sub_policys[domain][0];
15969                 sub_policy_idx = sub_policy->idx;
15970         } else {
15971                 sub_policy = mlx5_ipool_zmalloc
15972                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
15973                                 &sub_policy_idx);
15974                 if (!sub_policy ||
15975                         sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
15976                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
15977                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
15978                         goto rss_sub_policy_error;
15979                 }
15980                 sub_policy->idx = sub_policy_idx;
15981                 sub_policy->main_policy = mtr_policy;
15982         }
15983         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15984                 if (!rss_desc[i])
15985                         continue;
15986                 sub_policy->rix_hrxq[i] = hrxq_idx[i];
15987                 /*
15988                  * Overwrite the last action from
15989                  * RSS action to Queue action.
15990                  */
15991                 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
15992                               hrxq_idx[i]);
15993                 if (!hrxq) {
15994                         DRV_LOG(ERR, "Failed to create policy hrxq");
15995                         goto rss_sub_policy_error;
15996                 }
15997                 act_cnt = &mtr_policy->act_cnt[i];
15998                 if (act_cnt->rix_mark || act_cnt->modify_hdr) {
15999                         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
16000                         if (act_cnt->rix_mark)
16001                                 dh.mark = 1;
16002                         dh.fate_action = MLX5_FLOW_FATE_QUEUE;
16003                         dh.rix_hrxq = hrxq_idx[i];
16004                         flow_drv_rxq_flags_set(dev, &dh);
16005                 }
16006         }
16007         if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16008                 sub_policy, domain)) {
16009                 DRV_LOG(ERR, "Failed to create policy "
16010                         "rules per domain.");
16011                 goto rss_sub_policy_error;
16012         }
16013         if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16014                 i = (mtr_policy->sub_policy_num >>
16015                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16016                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16017                 mtr_policy->sub_policys[domain][i] = sub_policy;
16018                 i++;
16019                 if (i > MLX5_MTR_RSS_MAX_SUB_POLICY)
16020                         goto rss_sub_policy_error;
16021                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16022                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16023                 mtr_policy->sub_policy_num |=
16024                         (i & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16025                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16026         }
16027         rte_spinlock_unlock(&mtr_policy->sl);
16028         return sub_policy;
16029 rss_sub_policy_error:
16030         if (sub_policy) {
16031                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16032                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16033                         i = (mtr_policy->sub_policy_num >>
16034                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16035                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16036                         mtr_policy->sub_policys[domain][i] = NULL;
16037                         mlx5_ipool_free
16038                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16039                                         sub_policy->idx);
16040                 }
16041         }
16042         if (sub_policy_idx)
16043                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16044                         sub_policy_idx);
16045         rte_spinlock_unlock(&mtr_policy->sl);
16046         return NULL;
16047 }
16048
16049
16050 /**
16051  * Destroy the sub policy table with RX queue.
16052  *
16053  * @param[in] dev
16054  *   Pointer to Ethernet device.
16055  * @param[in] mtr_policy
16056  *   Pointer to meter policy table.
16057  */
16058 static void
16059 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
16060                 struct mlx5_flow_meter_policy *mtr_policy)
16061 {
16062         struct mlx5_priv *priv = dev->data->dev_private;
16063         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16064         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16065         uint32_t i, j;
16066         uint16_t sub_policy_num, new_policy_num;
16067
16068         rte_spinlock_lock(&mtr_policy->sl);
16069         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16070                 switch (mtr_policy->act_cnt[i].fate_action) {
16071                 case MLX5_FLOW_FATE_SHARED_RSS:
16072                         sub_policy_num = (mtr_policy->sub_policy_num >>
16073                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16074                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16075                         new_policy_num = sub_policy_num;
16076                         for (j = 0; j < sub_policy_num; j++) {
16077                                 sub_policy =
16078                                         mtr_policy->sub_policys[domain][j];
16079                                 if (sub_policy) {
16080                                         __flow_dv_destroy_sub_policy_rules(dev,
16081                                                 sub_policy);
16082                                 if (sub_policy !=
16083                                         mtr_policy->sub_policys[domain][0]) {
16084                                         mtr_policy->sub_policys[domain][j] =
16085                                                                 NULL;
16086                                         mlx5_ipool_free
16087                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16088                                                 sub_policy->idx);
16089                                                 new_policy_num--;
16090                                         }
16091                                 }
16092                         }
16093                         if (new_policy_num != sub_policy_num) {
16094                                 mtr_policy->sub_policy_num &=
16095                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16096                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16097                                 mtr_policy->sub_policy_num |=
16098                                 (new_policy_num &
16099                                         MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16100                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16101                         }
16102                         break;
16103                 case MLX5_FLOW_FATE_QUEUE:
16104                         sub_policy = mtr_policy->sub_policys[domain][0];
16105                         __flow_dv_destroy_sub_policy_rules(dev,
16106                                                 sub_policy);
16107                         break;
16108                 default:
16109                         /*Other actions without queue and do nothing*/
16110                         break;
16111                 }
16112         }
16113         rte_spinlock_unlock(&mtr_policy->sl);
16114 }
16115
16116 /**
16117  * Validate the batch counter support in root table.
16118  *
16119  * Create a simple flow with invalid counter and drop action on root table to
16120  * validate if batch counter with offset on root table is supported or not.
16121  *
16122  * @param[in] dev
16123  *   Pointer to rte_eth_dev structure.
16124  *
16125  * @return
16126  *   0 on success, a negative errno value otherwise and rte_errno is set.
16127  */
16128 int
16129 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
16130 {
16131         struct mlx5_priv *priv = dev->data->dev_private;
16132         struct mlx5_dev_ctx_shared *sh = priv->sh;
16133         struct mlx5_flow_dv_match_params mask = {
16134                 .size = sizeof(mask.buf),
16135         };
16136         struct mlx5_flow_dv_match_params value = {
16137                 .size = sizeof(value.buf),
16138         };
16139         struct mlx5dv_flow_matcher_attr dv_attr = {
16140                 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS,
16141                 .priority = 0,
16142                 .match_criteria_enable = 0,
16143                 .match_mask = (void *)&mask,
16144         };
16145         void *actions[2] = { 0 };
16146         struct mlx5_flow_tbl_resource *tbl = NULL;
16147         struct mlx5_devx_obj *dcs = NULL;
16148         void *matcher = NULL;
16149         void *flow = NULL;
16150         int ret = -1;
16151
16152         tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL,
16153                                         0, 0, 0, NULL);
16154         if (!tbl)
16155                 goto err;
16156         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
16157         if (!dcs)
16158                 goto err;
16159         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
16160                                                     &actions[0]);
16161         if (ret)
16162                 goto err;
16163         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
16164         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
16165                                                &matcher);
16166         if (ret)
16167                 goto err;
16168         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
16169                                        actions, &flow);
16170 err:
16171         /*
16172          * If batch counter with offset is not supported, the driver will not
16173          * validate the invalid offset value, flow create should success.
16174          * In this case, it means batch counter is not supported in root table.
16175          *
16176          * Otherwise, if flow create is failed, counter offset is supported.
16177          */
16178         if (flow) {
16179                 DRV_LOG(INFO, "Batch counter is not supported in root "
16180                               "table. Switch to fallback mode.");
16181                 rte_errno = ENOTSUP;
16182                 ret = -rte_errno;
16183                 claim_zero(mlx5_flow_os_destroy_flow(flow));
16184         } else {
16185                 /* Check matcher to make sure validate fail at flow create. */
16186                 if (!matcher || (matcher && errno != EINVAL))
16187                         DRV_LOG(ERR, "Unexpected error in counter offset "
16188                                      "support detection");
16189                 ret = 0;
16190         }
16191         if (actions[0])
16192                 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0]));
16193         if (matcher)
16194                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
16195         if (tbl)
16196                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
16197         if (dcs)
16198                 claim_zero(mlx5_devx_cmd_destroy(dcs));
16199         return ret;
16200 }
16201
16202 /**
16203  * Query a devx counter.
16204  *
16205  * @param[in] dev
16206  *   Pointer to the Ethernet device structure.
16207  * @param[in] cnt
16208  *   Index to the flow counter.
16209  * @param[in] clear
16210  *   Set to clear the counter statistics.
16211  * @param[out] pkts
16212  *   The statistics value of packets.
16213  * @param[out] bytes
16214  *   The statistics value of bytes.
16215  *
16216  * @return
16217  *   0 on success, otherwise return -1.
16218  */
16219 static int
16220 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
16221                       uint64_t *pkts, uint64_t *bytes)
16222 {
16223         struct mlx5_priv *priv = dev->data->dev_private;
16224         struct mlx5_flow_counter *cnt;
16225         uint64_t inn_pkts, inn_bytes;
16226         int ret;
16227
16228         if (!priv->config.devx)
16229                 return -1;
16230
16231         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
16232         if (ret)
16233                 return -1;
16234         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
16235         *pkts = inn_pkts - cnt->hits;
16236         *bytes = inn_bytes - cnt->bytes;
16237         if (clear) {
16238                 cnt->hits = inn_pkts;
16239                 cnt->bytes = inn_bytes;
16240         }
16241         return 0;
16242 }
16243
16244 /**
16245  * Get aged-out flows.
16246  *
16247  * @param[in] dev
16248  *   Pointer to the Ethernet device structure.
16249  * @param[in] context
16250  *   The address of an array of pointers to the aged-out flows contexts.
16251  * @param[in] nb_contexts
16252  *   The length of context array pointers.
16253  * @param[out] error
16254  *   Perform verbose error reporting if not NULL. Initialized in case of
16255  *   error only.
16256  *
16257  * @return
16258  *   how many contexts get in success, otherwise negative errno value.
16259  *   if nb_contexts is 0, return the amount of all aged contexts.
16260  *   if nb_contexts is not 0 , return the amount of aged flows reported
16261  *   in the context array.
16262  * @note: only stub for now
16263  */
16264 static int
16265 flow_get_aged_flows(struct rte_eth_dev *dev,
16266                     void **context,
16267                     uint32_t nb_contexts,
16268                     struct rte_flow_error *error)
16269 {
16270         struct mlx5_priv *priv = dev->data->dev_private;
16271         struct mlx5_age_info *age_info;
16272         struct mlx5_age_param *age_param;
16273         struct mlx5_flow_counter *counter;
16274         struct mlx5_aso_age_action *act;
16275         int nb_flows = 0;
16276
16277         if (nb_contexts && !context)
16278                 return rte_flow_error_set(error, EINVAL,
16279                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16280                                           NULL, "empty context");
16281         age_info = GET_PORT_AGE_INFO(priv);
16282         rte_spinlock_lock(&age_info->aged_sl);
16283         LIST_FOREACH(act, &age_info->aged_aso, next) {
16284                 nb_flows++;
16285                 if (nb_contexts) {
16286                         context[nb_flows - 1] =
16287                                                 act->age_params.context;
16288                         if (!(--nb_contexts))
16289                                 break;
16290                 }
16291         }
16292         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
16293                 nb_flows++;
16294                 if (nb_contexts) {
16295                         age_param = MLX5_CNT_TO_AGE(counter);
16296                         context[nb_flows - 1] = age_param->context;
16297                         if (!(--nb_contexts))
16298                                 break;
16299                 }
16300         }
16301         rte_spinlock_unlock(&age_info->aged_sl);
16302         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
16303         return nb_flows;
16304 }
16305
16306 /*
16307  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
16308  */
16309 static uint32_t
16310 flow_dv_counter_allocate(struct rte_eth_dev *dev)
16311 {
16312         return flow_dv_counter_alloc(dev, 0);
16313 }
16314
16315 /**
16316  * Validate indirect action.
16317  * Dispatcher for action type specific validation.
16318  *
16319  * @param[in] dev
16320  *   Pointer to the Ethernet device structure.
16321  * @param[in] conf
16322  *   Indirect action configuration.
16323  * @param[in] action
16324  *   The indirect action object to validate.
16325  * @param[out] error
16326  *   Perform verbose error reporting if not NULL. Initialized in case of
16327  *   error only.
16328  *
16329  * @return
16330  *   0 on success, otherwise negative errno value.
16331  */
16332 static int
16333 flow_dv_action_validate(struct rte_eth_dev *dev,
16334                         const struct rte_flow_indir_action_conf *conf,
16335                         const struct rte_flow_action *action,
16336                         struct rte_flow_error *err)
16337 {
16338         struct mlx5_priv *priv = dev->data->dev_private;
16339
16340         RTE_SET_USED(conf);
16341         switch (action->type) {
16342         case RTE_FLOW_ACTION_TYPE_RSS:
16343                 /*
16344                  * priv->obj_ops is set according to driver capabilities.
16345                  * When DevX capabilities are
16346                  * sufficient, it is set to devx_obj_ops.
16347                  * Otherwise, it is set to ibv_obj_ops.
16348                  * ibv_obj_ops doesn't support ind_table_modify operation.
16349                  * In this case the indirect RSS action can't be used.
16350                  */
16351                 if (priv->obj_ops.ind_table_modify == NULL)
16352                         return rte_flow_error_set
16353                                         (err, ENOTSUP,
16354                                          RTE_FLOW_ERROR_TYPE_ACTION,
16355                                          NULL,
16356                                          "Indirect RSS action not supported");
16357                 return mlx5_validate_action_rss(dev, action, err);
16358         case RTE_FLOW_ACTION_TYPE_AGE:
16359                 if (!priv->sh->aso_age_mng)
16360                         return rte_flow_error_set(err, ENOTSUP,
16361                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16362                                                 NULL,
16363                                                 "Indirect age action not supported");
16364                 return flow_dv_validate_action_age(0, action, dev, err);
16365         case RTE_FLOW_ACTION_TYPE_COUNT:
16366                 /*
16367                  * There are two mechanisms to share the action count.
16368                  * The old mechanism uses the shared field to share, while the
16369                  * new mechanism uses the indirect action API.
16370                  * This validation comes to make sure that the two mechanisms
16371                  * are not combined.
16372                  */
16373                 if (is_shared_action_count(action))
16374                         return rte_flow_error_set(err, ENOTSUP,
16375                                                   RTE_FLOW_ERROR_TYPE_ACTION,
16376                                                   NULL,
16377                                                   "Mix shared and indirect counter is not supported");
16378                 return flow_dv_validate_action_count(dev, true, 0, err);
16379         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
16380                 if (!priv->sh->ct_aso_en)
16381                         return rte_flow_error_set(err, ENOTSUP,
16382                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
16383                                         "ASO CT is not supported");
16384                 return mlx5_validate_action_ct(dev, action->conf, err);
16385         default:
16386                 return rte_flow_error_set(err, ENOTSUP,
16387                                           RTE_FLOW_ERROR_TYPE_ACTION,
16388                                           NULL,
16389                                           "action type not supported");
16390         }
16391 }
16392
16393 /**
16394  * Validate meter policy actions.
16395  * Dispatcher for action type specific validation.
16396  *
16397  * @param[in] dev
16398  *   Pointer to the Ethernet device structure.
16399  * @param[in] action
16400  *   The meter policy action object to validate.
16401  * @param[in] attr
16402  *   Attributes of flow to determine steering domain.
16403  * @param[out] error
16404  *   Perform verbose error reporting if not NULL. Initialized in case of
16405  *   error only.
16406  *
16407  * @return
16408  *   0 on success, otherwise negative errno value.
16409  */
16410 static int
16411 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
16412                         const struct rte_flow_action *actions[RTE_COLORS],
16413                         struct rte_flow_attr *attr,
16414                         bool *is_rss,
16415                         uint8_t *domain_bitmap,
16416                         bool *is_def_policy,
16417                         struct rte_mtr_error *error)
16418 {
16419         struct mlx5_priv *priv = dev->data->dev_private;
16420         struct mlx5_dev_config *dev_conf = &priv->config;
16421         const struct rte_flow_action *act;
16422         uint64_t action_flags = 0;
16423         int actions_n;
16424         int i, ret;
16425         struct rte_flow_error flow_err;
16426         uint8_t domain_color[RTE_COLORS] = {0};
16427         uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
16428
16429         if (!priv->config.dv_esw_en)
16430                 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
16431         *domain_bitmap = def_domain;
16432         if (actions[RTE_COLOR_YELLOW] &&
16433                 actions[RTE_COLOR_YELLOW]->type != RTE_FLOW_ACTION_TYPE_END)
16434                 return -rte_mtr_error_set(error, ENOTSUP,
16435                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
16436                                 NULL,
16437                                 "Yellow color does not support any action.");
16438         if (actions[RTE_COLOR_YELLOW] &&
16439                 actions[RTE_COLOR_YELLOW]->type != RTE_FLOW_ACTION_TYPE_DROP)
16440                 return -rte_mtr_error_set(error, ENOTSUP,
16441                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
16442                                 NULL, "Red color only supports drop action.");
16443         /*
16444          * Check default policy actions:
16445          * Green/Yellow: no action, Red: drop action
16446          */
16447         if ((!actions[RTE_COLOR_GREEN] ||
16448                 actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END)) {
16449                 *is_def_policy = true;
16450                 return 0;
16451         }
16452         flow_err.message = NULL;
16453         for (i = 0; i < RTE_COLORS; i++) {
16454                 act = actions[i];
16455                 for (action_flags = 0, actions_n = 0;
16456                         act && act->type != RTE_FLOW_ACTION_TYPE_END;
16457                         act++) {
16458                         if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
16459                                 return -rte_mtr_error_set(error, ENOTSUP,
16460                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16461                                           NULL, "too many actions");
16462                         switch (act->type) {
16463                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
16464                                 if (!priv->config.dv_esw_en)
16465                                         return -rte_mtr_error_set(error,
16466                                         ENOTSUP,
16467                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16468                                         NULL, "PORT action validate check"
16469                                         " fail for ESW disable");
16470                                 ret = flow_dv_validate_action_port_id(dev,
16471                                                 action_flags,
16472                                                 act, attr, &flow_err);
16473                                 if (ret)
16474                                         return -rte_mtr_error_set(error,
16475                                         ENOTSUP,
16476                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16477                                         NULL, flow_err.message ?
16478                                         flow_err.message :
16479                                         "PORT action validate check fail");
16480                                 ++actions_n;
16481                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
16482                                 break;
16483                         case RTE_FLOW_ACTION_TYPE_MARK:
16484                                 ret = flow_dv_validate_action_mark(dev, act,
16485                                                            action_flags,
16486                                                            attr, &flow_err);
16487                                 if (ret < 0)
16488                                         return -rte_mtr_error_set(error,
16489                                         ENOTSUP,
16490                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16491                                         NULL, flow_err.message ?
16492                                         flow_err.message :
16493                                         "Mark action validate check fail");
16494                                 if (dev_conf->dv_xmeta_en !=
16495                                         MLX5_XMETA_MODE_LEGACY)
16496                                         return -rte_mtr_error_set(error,
16497                                         ENOTSUP,
16498                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16499                                         NULL, "Extend MARK action is "
16500                                         "not supported. Please try use "
16501                                         "default policy for meter.");
16502                                 action_flags |= MLX5_FLOW_ACTION_MARK;
16503                                 ++actions_n;
16504                                 break;
16505                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
16506                                 ret = flow_dv_validate_action_set_tag(dev,
16507                                                         act, action_flags,
16508                                                         attr, &flow_err);
16509                                 if (ret)
16510                                         return -rte_mtr_error_set(error,
16511                                         ENOTSUP,
16512                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16513                                         NULL, flow_err.message ?
16514                                         flow_err.message :
16515                                         "Set tag action validate check fail");
16516                                 /*
16517                                  * Count all modify-header actions
16518                                  * as one action.
16519                                  */
16520                                 if (!(action_flags &
16521                                         MLX5_FLOW_MODIFY_HDR_ACTIONS))
16522                                         ++actions_n;
16523                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
16524                                 break;
16525                         case RTE_FLOW_ACTION_TYPE_DROP:
16526                                 ret = mlx5_flow_validate_action_drop
16527                                         (action_flags,
16528                                         attr, &flow_err);
16529                                 if (ret < 0)
16530                                         return -rte_mtr_error_set(error,
16531                                         ENOTSUP,
16532                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16533                                         NULL, flow_err.message ?
16534                                         flow_err.message :
16535                                         "Drop action validate check fail");
16536                                 action_flags |= MLX5_FLOW_ACTION_DROP;
16537                                 ++actions_n;
16538                                 break;
16539                         case RTE_FLOW_ACTION_TYPE_QUEUE:
16540                                 /*
16541                                  * Check whether extensive
16542                                  * metadata feature is engaged.
16543                                  */
16544                                 if (dev_conf->dv_flow_en &&
16545                                         (dev_conf->dv_xmeta_en !=
16546                                         MLX5_XMETA_MODE_LEGACY) &&
16547                                         mlx5_flow_ext_mreg_supported(dev))
16548                                         return -rte_mtr_error_set(error,
16549                                           ENOTSUP,
16550                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16551                                           NULL, "Queue action with meta "
16552                                           "is not supported. Please try use "
16553                                           "default policy for meter.");
16554                                 ret = mlx5_flow_validate_action_queue(act,
16555                                                         action_flags, dev,
16556                                                         attr, &flow_err);
16557                                 if (ret < 0)
16558                                         return -rte_mtr_error_set(error,
16559                                           ENOTSUP,
16560                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16561                                           NULL, flow_err.message ?
16562                                           flow_err.message :
16563                                           "Queue action validate check fail");
16564                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
16565                                 ++actions_n;
16566                                 break;
16567                         case RTE_FLOW_ACTION_TYPE_RSS:
16568                                 if (dev_conf->dv_flow_en &&
16569                                         (dev_conf->dv_xmeta_en !=
16570                                         MLX5_XMETA_MODE_LEGACY) &&
16571                                         mlx5_flow_ext_mreg_supported(dev))
16572                                         return -rte_mtr_error_set(error,
16573                                           ENOTSUP,
16574                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16575                                           NULL, "RSS action with meta "
16576                                           "is not supported. Please try use "
16577                                           "default policy for meter.");
16578                                 ret = mlx5_validate_action_rss(dev, act,
16579                                                 &flow_err);
16580                                 if (ret < 0)
16581                                         return -rte_mtr_error_set(error,
16582                                           ENOTSUP,
16583                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16584                                           NULL, flow_err.message ?
16585                                           flow_err.message :
16586                                           "RSS action validate check fail");
16587                                 action_flags |= MLX5_FLOW_ACTION_RSS;
16588                                 ++actions_n;
16589                                 *is_rss = true;
16590                                 break;
16591                         case RTE_FLOW_ACTION_TYPE_JUMP:
16592                                 ret = flow_dv_validate_action_jump(dev,
16593                                         NULL, act, action_flags,
16594                                         attr, true, &flow_err);
16595                                 if (ret)
16596                                         return -rte_mtr_error_set(error,
16597                                           ENOTSUP,
16598                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16599                                           NULL, flow_err.message ?
16600                                           flow_err.message :
16601                                           "Jump action validate check fail");
16602                                 ++actions_n;
16603                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
16604                                 break;
16605                         default:
16606                                 return -rte_mtr_error_set(error, ENOTSUP,
16607                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16608                                         NULL,
16609                                         "Doesn't support optional action");
16610                         }
16611                 }
16612                 /* Yellow is not supported, just skip. */
16613                 if (i == RTE_COLOR_YELLOW)
16614                         continue;
16615                 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
16616                         domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT;
16617                 else if ((action_flags &
16618                         (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) ||
16619                         (action_flags & MLX5_FLOW_ACTION_MARK))
16620                         /*
16621                          * Only support MLX5_XMETA_MODE_LEGACY
16622                          * so MARK action only in ingress domain.
16623                          */
16624                         domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT;
16625                 else
16626                         domain_color[i] = def_domain;
16627                 /*
16628                  * Validate the drop action mutual exclusion
16629                  * with other actions. Drop action is mutually-exclusive
16630                  * with any other action, except for Count action.
16631                  */
16632                 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
16633                         (action_flags & ~MLX5_FLOW_ACTION_DROP)) {
16634                         return -rte_mtr_error_set(error, ENOTSUP,
16635                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
16636                                 NULL, "Drop action is mutually-exclusive "
16637                                 "with any other action");
16638                 }
16639                 /* Eswitch has few restrictions on using items and actions */
16640                 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) {
16641                         if (!mlx5_flow_ext_mreg_supported(dev) &&
16642                                 action_flags & MLX5_FLOW_ACTION_MARK)
16643                                 return -rte_mtr_error_set(error, ENOTSUP,
16644                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16645                                         NULL, "unsupported action MARK");
16646                         if (action_flags & MLX5_FLOW_ACTION_QUEUE)
16647                                 return -rte_mtr_error_set(error, ENOTSUP,
16648                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16649                                         NULL, "unsupported action QUEUE");
16650                         if (action_flags & MLX5_FLOW_ACTION_RSS)
16651                                 return -rte_mtr_error_set(error, ENOTSUP,
16652                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16653                                         NULL, "unsupported action RSS");
16654                         if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
16655                                 return -rte_mtr_error_set(error, ENOTSUP,
16656                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16657                                         NULL, "no fate action is found");
16658                 } else {
16659                         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) &&
16660                                 (domain_color[i] &
16661                                 MLX5_MTR_DOMAIN_INGRESS_BIT)) {
16662                                 if ((domain_color[i] &
16663                                         MLX5_MTR_DOMAIN_EGRESS_BIT))
16664                                         domain_color[i] =
16665                                         MLX5_MTR_DOMAIN_EGRESS_BIT;
16666                                 else
16667                                         return -rte_mtr_error_set(error,
16668                                         ENOTSUP,
16669                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16670                                         NULL, "no fate action is found");
16671                         }
16672                 }
16673                 if (domain_color[i] != def_domain)
16674                         *domain_bitmap = domain_color[i];
16675         }
16676         return 0;
16677 }
16678
16679 static int
16680 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
16681 {
16682         struct mlx5_priv *priv = dev->data->dev_private;
16683         int ret = 0;
16684
16685         if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
16686                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain,
16687                                                 flags);
16688                 if (ret != 0)
16689                         return ret;
16690         }
16691         if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
16692                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags);
16693                 if (ret != 0)
16694                         return ret;
16695         }
16696         if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
16697                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags);
16698                 if (ret != 0)
16699                         return ret;
16700         }
16701         return 0;
16702 }
16703
16704 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
16705         .validate = flow_dv_validate,
16706         .prepare = flow_dv_prepare,
16707         .translate = flow_dv_translate,
16708         .apply = flow_dv_apply,
16709         .remove = flow_dv_remove,
16710         .destroy = flow_dv_destroy,
16711         .query = flow_dv_query,
16712         .create_mtr_tbls = flow_dv_create_mtr_tbls,
16713         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls,
16714         .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls,
16715         .create_meter = flow_dv_mtr_alloc,
16716         .free_meter = flow_dv_aso_mtr_release_to_pool,
16717         .validate_mtr_acts = flow_dv_validate_mtr_policy_acts,
16718         .create_mtr_acts = flow_dv_create_mtr_policy_acts,
16719         .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts,
16720         .create_policy_rules = flow_dv_create_policy_rules,
16721         .destroy_policy_rules = flow_dv_destroy_policy_rules,
16722         .create_def_policy = flow_dv_create_def_policy,
16723         .destroy_def_policy = flow_dv_destroy_def_policy,
16724         .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare,
16725         .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq,
16726         .counter_alloc = flow_dv_counter_allocate,
16727         .counter_free = flow_dv_counter_free,
16728         .counter_query = flow_dv_counter_query,
16729         .get_aged_flows = flow_get_aged_flows,
16730         .action_validate = flow_dv_action_validate,
16731         .action_create = flow_dv_action_create,
16732         .action_destroy = flow_dv_action_destroy,
16733         .action_update = flow_dv_action_update,
16734         .action_query = flow_dv_action_query,
16735         .sync_domain = flow_dv_sync_domain,
16736 };
16737
16738 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
16739