drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_bus_pci.h>
  19 #include <rte_ip.h>
  20 #include <rte_gre.h>
  21 #include <rte_vxlan.h>
  22 #include <rte_gtp.h>
  23 #include <rte_eal_paging.h>
  24 #include <rte_mpls.h>
  25 #include <rte_mtr.h>
  26 #include <rte_mtr_driver.h>
  27 #include <rte_tailq.h>
  28
  29 #include <mlx5_glue.h>
  30 #include <mlx5_devx_cmds.h>
  31 #include <mlx5_prm.h>
  32 #include <mlx5_malloc.h>
  33
  34 #include "mlx5_defs.h"
  35 #include "mlx5.h"
  36 #include "mlx5_common_os.h"
  37 #include "mlx5_flow.h"
  38 #include "mlx5_flow_os.h"
  39 #include "mlx5_rx.h"
  40 #include "mlx5_tx.h"
  41 #include "rte_pmd_mlx5.h"
  42
  43 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  44
  45 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  46 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR_ESWITCH
  50 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  51 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  52 #endif
  53 #endif
  54
  55 #ifndef HAVE_MLX5DV_DR
  56 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  57 #endif
  58
  59 /* VLAN header definitions */
  60 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  61 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  62 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  63 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  64 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  65
  66 union flow_dv_attr {
  67         struct {
  68                 uint32_t valid:1;
  69                 uint32_t ipv4:1;
  70                 uint32_t ipv6:1;
  71                 uint32_t tcp:1;
  72                 uint32_t udp:1;
  73                 uint32_t reserved:27;
  74         };
  75         uint32_t attr;
  76 };
  77
  78 static int
  79 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
  80                              struct mlx5_flow_tbl_resource *tbl);
  81
  82 static int
  83 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  84                                      uint32_t encap_decap_idx);
  85
  86 static int
  87 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  88                                         uint32_t port_id);
  89 static void
  90 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);
  91
  92 static int
  93 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
  94                                   uint32_t rix_jump);
  95
  96 static int16_t
  97 flow_dv_get_esw_manager_vport_id(struct rte_eth_dev *dev)
  98 {
  99         struct mlx5_priv *priv = dev->data->dev_private;
 100         struct mlx5_common_device *cdev = priv->sh->cdev;
 101
 102         if (cdev->config.hca_attr.esw_mgr_vport_id_valid)
 103                 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id;
 104
 105         if (priv->pci_dev == NULL)
 106                 return 0;
 107         switch (priv->pci_dev->id.device_id) {
 108         case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF:
 109         case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF:
 110         case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF:
 111                 return (int16_t)0xfffe;
 112         default:
 113                 return 0;
 114         }
 115 }
 116
 117 /**
 118  * Initialize flow attributes structure according to flow items' types.
 119  *
 120  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
 121  * mode. For tunnel mode, the items to be modified are the outermost ones.
 122  *
 123  * @param[in] item
 124  *   Pointer to item specification.
 125  * @param[out] attr
 126  *   Pointer to flow attributes structure.
 127  * @param[in] dev_flow
 128  *   Pointer to the sub flow.
 129  * @param[in] tunnel_decap
 130  *   Whether action is after tunnel decapsulation.
 131  */
 132 static void
 133 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 134                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 135 {
 136         uint64_t layers = dev_flow->handle->layers;
 137
 138         /*
 139          * If layers is already initialized, it means this dev_flow is the
 140          * suffix flow, the layers flags is set by the prefix flow. Need to
 141          * use the layer flags from prefix flow as the suffix flow may not
 142          * have the user defined items as the flow is split.
 143          */
 144         if (layers) {
 145                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 146                         attr->ipv4 = 1;
 147                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 148                         attr->ipv6 = 1;
 149                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 150                         attr->tcp = 1;
 151                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 152                         attr->udp = 1;
 153                 attr->valid = 1;
 154                 return;
 155         }
 156         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 157                 uint8_t next_protocol = 0xff;
 158                 switch (item->type) {
 159                 case RTE_FLOW_ITEM_TYPE_GRE:
 160                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 161                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 162                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 163                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 164                 case RTE_FLOW_ITEM_TYPE_MPLS:
 165                 case RTE_FLOW_ITEM_TYPE_GTP:
 166                         if (tunnel_decap)
 167                                 attr->attr = 0;
 168                         break;
 169                 case RTE_FLOW_ITEM_TYPE_IPV4:
 170                         if (!attr->ipv6)
 171                                 attr->ipv4 = 1;
 172                         if (item->mask != NULL &&
 173                             ((const struct rte_flow_item_ipv4 *)
 174                             item->mask)->hdr.next_proto_id)
 175                                 next_protocol =
 176                                     ((const struct rte_flow_item_ipv4 *)
 177                                       (item->spec))->hdr.next_proto_id &
 178                                     ((const struct rte_flow_item_ipv4 *)
 179                                       (item->mask))->hdr.next_proto_id;
 180                         if ((next_protocol == IPPROTO_IPIP ||
 181                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 182                                 attr->attr = 0;
 183                         break;
 184                 case RTE_FLOW_ITEM_TYPE_IPV6:
 185                         if (!attr->ipv4)
 186                                 attr->ipv6 = 1;
 187                         if (item->mask != NULL &&
 188                             ((const struct rte_flow_item_ipv6 *)
 189                             item->mask)->hdr.proto)
 190                                 next_protocol =
 191                                     ((const struct rte_flow_item_ipv6 *)
 192                                       (item->spec))->hdr.proto &
 193                                     ((const struct rte_flow_item_ipv6 *)
 194                                       (item->mask))->hdr.proto;
 195                         if ((next_protocol == IPPROTO_IPIP ||
 196                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 197                                 attr->attr = 0;
 198                         break;
 199                 case RTE_FLOW_ITEM_TYPE_UDP:
 200                         if (!attr->tcp)
 201                                 attr->udp = 1;
 202                         break;
 203                 case RTE_FLOW_ITEM_TYPE_TCP:
 204                         if (!attr->udp)
 205                                 attr->tcp = 1;
 206                         break;
 207                 default:
 208                         break;
 209                 }
 210         }
 211         attr->valid = 1;
 212 }
 213
 214 /*
 215  * Convert rte_mtr_color to mlx5 color.
 216  *
 217  * @param[in] rcol
 218  *   rte_mtr_color.
 219  *
 220  * @return
 221  *   mlx5 color.
 222  */
 223 static inline int
 224 rte_col_2_mlx5_col(enum rte_color rcol)
 225 {
 226         switch (rcol) {
 227         case RTE_COLOR_GREEN:
 228                 return MLX5_FLOW_COLOR_GREEN;
 229         case RTE_COLOR_YELLOW:
 230                 return MLX5_FLOW_COLOR_YELLOW;
 231         case RTE_COLOR_RED:
 232                 return MLX5_FLOW_COLOR_RED;
 233         default:
 234                 break;
 235         }
 236         return MLX5_FLOW_COLOR_UNDEFINED;
 237 }
 238
 239 struct field_modify_info {
 240         uint32_t size; /* Size of field in protocol header, in bytes. */
 241         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 242         enum mlx5_modification_field id;
 243 };
 244
 245 struct field_modify_info modify_eth[] = {
 246         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 247         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 248         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 249         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 250         {0, 0, 0},
 251 };
 252
 253 struct field_modify_info modify_vlan_out_first_vid[] = {
 254         /* Size in bits !!! */
 255         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 256         {0, 0, 0},
 257 };
 258
 259 struct field_modify_info modify_ipv4[] = {
 260         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 261         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 262         {4, 12, MLX5_MODI_OUT_SIPV4},
 263         {4, 16, MLX5_MODI_OUT_DIPV4},
 264         {0, 0, 0},
 265 };
 266
 267 struct field_modify_info modify_ipv6[] = {
 268         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 269         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 270         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 271         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 272         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 273         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 274         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 275         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 276         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 277         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 278         {0, 0, 0},
 279 };
 280
 281 struct field_modify_info modify_udp[] = {
 282         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 283         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 284         {0, 0, 0},
 285 };
 286
 287 struct field_modify_info modify_tcp[] = {
 288         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 289         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 290         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 291         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 292         {0, 0, 0},
 293 };
 294
 295 static void
 296 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 297                           uint8_t next_protocol, uint64_t *item_flags,
 298                           int *tunnel)
 299 {
 300         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 301                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 302         if (next_protocol == IPPROTO_IPIP) {
 303                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 304                 *tunnel = 1;
 305         }
 306         if (next_protocol == IPPROTO_IPV6) {
 307                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 308                 *tunnel = 1;
 309         }
 310 }
 311
 312 static inline struct mlx5_hlist *
 313 flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
 314                      const char *name, uint32_t size, bool direct_key,
 315                      bool lcores_share, void *ctx,
 316                      mlx5_list_create_cb cb_create,
 317                      mlx5_list_match_cb cb_match,
 318                      mlx5_list_remove_cb cb_remove,
 319                      mlx5_list_clone_cb cb_clone,
 320                      mlx5_list_clone_free_cb cb_clone_free,
 321                      struct rte_flow_error *error)
 322 {
 323         struct mlx5_hlist *hl;
 324         struct mlx5_hlist *expected = NULL;
 325         char s[MLX5_NAME_SIZE];
 326
 327         hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 328         if (likely(hl))
 329                 return hl;
 330         snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
 331         hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
 332                         ctx, cb_create, cb_match, cb_remove, cb_clone,
 333                         cb_clone_free);
 334         if (!hl) {
 335                 DRV_LOG(ERR, "%s hash creation failed", name);
 336                 rte_flow_error_set(error, ENOMEM,
 337                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 338                                    "cannot allocate resource memory");
 339                 return NULL;
 340         }
 341         if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
 342                                          __ATOMIC_SEQ_CST,
 343                                          __ATOMIC_SEQ_CST)) {
 344                 mlx5_hlist_destroy(hl);
 345                 hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 346         }
 347         return hl;
 348 }
 349
 350 /* Update VLAN's VID/PCP based on input rte_flow_action.
 351  *
 352  * @param[in] action
 353  *   Pointer to struct rte_flow_action.
 354  * @param[out] vlan
 355  *   Pointer to struct rte_vlan_hdr.
 356  */
 357 static void
 358 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 359                          struct rte_vlan_hdr *vlan)
 360 {
 361         uint16_t vlan_tci;
 362         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 363                 vlan_tci =
 364                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 365                                                action->conf)->vlan_pcp;
 366                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 367                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 368                 vlan->vlan_tci |= vlan_tci;
 369         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 370                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 371                 vlan->vlan_tci |= rte_be_to_cpu_16
 372                     (((const struct rte_flow_action_of_set_vlan_vid *)
 373                                              action->conf)->vlan_vid);
 374         }
 375 }
 376
 377 /**
 378  * Fetch 1, 2, 3 or 4 byte field from the byte array
 379  * and return as unsigned integer in host-endian format.
 380  *
 381  * @param[in] data
 382  *   Pointer to data array.
 383  * @param[in] size
 384  *   Size of field to extract.
 385  *
 386  * @return
 387  *   converted field in host endian format.
 388  */
 389 static inline uint32_t
 390 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 391 {
 392         uint32_t ret;
 393
 394         switch (size) {
 395         case 1:
 396                 ret = *data;
 397                 break;
 398         case 2:
 399                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 400                 break;
 401         case 3:
 402                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 403                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 404                 break;
 405         case 4:
 406                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 407                 break;
 408         default:
 409                 MLX5_ASSERT(false);
 410                 ret = 0;
 411                 break;
 412         }
 413         return ret;
 414 }
 415
 416 /**
 417  * Convert modify-header action to DV specification.
 418  *
 419  * Data length of each action is determined by provided field description
 420  * and the item mask. Data bit offset and width of each action is determined
 421  * by provided item mask.
 422  *
 423  * @param[in] item
 424  *   Pointer to item specification.
 425  * @param[in] field
 426  *   Pointer to field modification information.
 427  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 428  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 429  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 430  * @param[in] dcopy
 431  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 432  *   Negative offset value sets the same offset as source offset.
 433  *   size field is ignored, value is taken from source field.
 434  * @param[in,out] resource
 435  *   Pointer to the modify-header resource.
 436  * @param[in] type
 437  *   Type of modification.
 438  * @param[out] error
 439  *   Pointer to the error structure.
 440  *
 441  * @return
 442  *   0 on success, a negative errno value otherwise and rte_errno is set.
 443  */
 444 static int
 445 flow_dv_convert_modify_action(struct rte_flow_item *item,
 446                               struct field_modify_info *field,
 447                               struct field_modify_info *dcopy,
 448                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 449                               uint32_t type, struct rte_flow_error *error)
 450 {
 451         uint32_t i = resource->actions_num;
 452         struct mlx5_modification_cmd *actions = resource->actions;
 453         uint32_t carry_b = 0;
 454
 455         /*
 456          * The item and mask are provided in big-endian format.
 457          * The fields should be presented as in big-endian format either.
 458          * Mask must be always present, it defines the actual field width.
 459          */
 460         MLX5_ASSERT(item->mask);
 461         MLX5_ASSERT(field->size);
 462         do {
 463                 uint32_t size_b;
 464                 uint32_t off_b;
 465                 uint32_t mask;
 466                 uint32_t data;
 467                 bool next_field = true;
 468                 bool next_dcopy = true;
 469
 470                 if (i >= MLX5_MAX_MODIFY_NUM)
 471                         return rte_flow_error_set(error, EINVAL,
 472                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 473                                  "too many items to modify");
 474                 /* Fetch variable byte size mask from the array. */
 475                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 476                                            field->offset, field->size);
 477                 if (!mask) {
 478                         ++field;
 479                         continue;
 480                 }
 481                 /* Deduce actual data width in bits from mask value. */
 482                 off_b = rte_bsf32(mask) + carry_b;
 483                 size_b = sizeof(uint32_t) * CHAR_BIT -
 484                          off_b - __builtin_clz(mask);
 485                 MLX5_ASSERT(size_b);
 486                 actions[i] = (struct mlx5_modification_cmd) {
 487                         .action_type = type,
 488                         .field = field->id,
 489                         .offset = off_b,
 490                         .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ?
 491                                 0 : size_b,
 492                 };
 493                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 494                         MLX5_ASSERT(dcopy);
 495                         actions[i].dst_field = dcopy->id;
 496                         actions[i].dst_offset =
 497                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 498                         /* Convert entire record to big-endian format. */
 499                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 500                         /*
 501                          * Destination field overflow. Copy leftovers of
 502                          * a source field to the next destination field.
 503                          */
 504                         carry_b = 0;
 505                         if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) &&
 506                             dcopy->size != 0) {
 507                                 actions[i].length =
 508                                         dcopy->size * CHAR_BIT - dcopy->offset;
 509                                 carry_b = actions[i].length;
 510                                 next_field = false;
 511                         }
 512                         /*
 513                          * Not enough bits in a source filed to fill a
 514                          * destination field. Switch to the next source.
 515                          */
 516                         if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) &&
 517                             (size_b == field->size * CHAR_BIT - off_b)) {
 518                                 actions[i].length =
 519                                         field->size * CHAR_BIT - off_b;
 520                                 dcopy->offset += actions[i].length;
 521                                 next_dcopy = false;
 522                         }
 523                         if (next_dcopy)
 524                                 ++dcopy;
 525                 } else {
 526                         MLX5_ASSERT(item->spec);
 527                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 528                                                    field->offset, field->size);
 529                         /* Shift out the trailing masked bits from data. */
 530                         data = (data & mask) >> off_b;
 531                         actions[i].data1 = rte_cpu_to_be_32(data);
 532                 }
 533                 /* Convert entire record to expected big-endian format. */
 534                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 535                 if (next_field)
 536                         ++field;
 537                 ++i;
 538         } while (field->size);
 539         if (resource->actions_num == i)
 540                 return rte_flow_error_set(error, EINVAL,
 541                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 542                                           "invalid modification flow item");
 543         resource->actions_num = i;
 544         return 0;
 545 }
 546
 547 /**
 548  * Convert modify-header set IPv4 address action to DV specification.
 549  *
 550  * @param[in,out] resource
 551  *   Pointer to the modify-header resource.
 552  * @param[in] action
 553  *   Pointer to action specification.
 554  * @param[out] error
 555  *   Pointer to the error structure.
 556  *
 557  * @return
 558  *   0 on success, a negative errno value otherwise and rte_errno is set.
 559  */
 560 static int
 561 flow_dv_convert_action_modify_ipv4
 562                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 563                          const struct rte_flow_action *action,
 564                          struct rte_flow_error *error)
 565 {
 566         const struct rte_flow_action_set_ipv4 *conf =
 567                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 568         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 569         struct rte_flow_item_ipv4 ipv4;
 570         struct rte_flow_item_ipv4 ipv4_mask;
 571
 572         memset(&ipv4, 0, sizeof(ipv4));
 573         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 574         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 575                 ipv4.hdr.src_addr = conf->ipv4_addr;
 576                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 577         } else {
 578                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 579                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 580         }
 581         item.spec = &ipv4;
 582         item.mask = &ipv4_mask;
 583         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 584                                              MLX5_MODIFICATION_TYPE_SET, error);
 585 }
 586
 587 /**
 588  * Convert modify-header set IPv6 address action to DV specification.
 589  *
 590  * @param[in,out] resource
 591  *   Pointer to the modify-header resource.
 592  * @param[in] action
 593  *   Pointer to action specification.
 594  * @param[out] error
 595  *   Pointer to the error structure.
 596  *
 597  * @return
 598  *   0 on success, a negative errno value otherwise and rte_errno is set.
 599  */
 600 static int
 601 flow_dv_convert_action_modify_ipv6
 602                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 603                          const struct rte_flow_action *action,
 604                          struct rte_flow_error *error)
 605 {
 606         const struct rte_flow_action_set_ipv6 *conf =
 607                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 608         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 609         struct rte_flow_item_ipv6 ipv6;
 610         struct rte_flow_item_ipv6 ipv6_mask;
 611
 612         memset(&ipv6, 0, sizeof(ipv6));
 613         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 614         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 615                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 616                        sizeof(ipv6.hdr.src_addr));
 617                 memcpy(&ipv6_mask.hdr.src_addr,
 618                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 619                        sizeof(ipv6.hdr.src_addr));
 620         } else {
 621                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 622                        sizeof(ipv6.hdr.dst_addr));
 623                 memcpy(&ipv6_mask.hdr.dst_addr,
 624                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 625                        sizeof(ipv6.hdr.dst_addr));
 626         }
 627         item.spec = &ipv6;
 628         item.mask = &ipv6_mask;
 629         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 630                                              MLX5_MODIFICATION_TYPE_SET, error);
 631 }
 632
 633 /**
 634  * Convert modify-header set MAC address action to DV specification.
 635  *
 636  * @param[in,out] resource
 637  *   Pointer to the modify-header resource.
 638  * @param[in] action
 639  *   Pointer to action specification.
 640  * @param[out] error
 641  *   Pointer to the error structure.
 642  *
 643  * @return
 644  *   0 on success, a negative errno value otherwise and rte_errno is set.
 645  */
 646 static int
 647 flow_dv_convert_action_modify_mac
 648                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 649                          const struct rte_flow_action *action,
 650                          struct rte_flow_error *error)
 651 {
 652         const struct rte_flow_action_set_mac *conf =
 653                 (const struct rte_flow_action_set_mac *)(action->conf);
 654         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 655         struct rte_flow_item_eth eth;
 656         struct rte_flow_item_eth eth_mask;
 657
 658         memset(&eth, 0, sizeof(eth));
 659         memset(&eth_mask, 0, sizeof(eth_mask));
 660         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 661                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 662                        sizeof(eth.src.addr_bytes));
 663                 memcpy(&eth_mask.src.addr_bytes,
 664                        &rte_flow_item_eth_mask.src.addr_bytes,
 665                        sizeof(eth_mask.src.addr_bytes));
 666         } else {
 667                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 668                        sizeof(eth.dst.addr_bytes));
 669                 memcpy(&eth_mask.dst.addr_bytes,
 670                        &rte_flow_item_eth_mask.dst.addr_bytes,
 671                        sizeof(eth_mask.dst.addr_bytes));
 672         }
 673         item.spec = &eth;
 674         item.mask = &eth_mask;
 675         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 676                                              MLX5_MODIFICATION_TYPE_SET, error);
 677 }
 678
 679 /**
 680  * Convert modify-header set VLAN VID action to DV specification.
 681  *
 682  * @param[in,out] resource
 683  *   Pointer to the modify-header resource.
 684  * @param[in] action
 685  *   Pointer to action specification.
 686  * @param[out] error
 687  *   Pointer to the error structure.
 688  *
 689  * @return
 690  *   0 on success, a negative errno value otherwise and rte_errno is set.
 691  */
 692 static int
 693 flow_dv_convert_action_modify_vlan_vid
 694                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 695                          const struct rte_flow_action *action,
 696                          struct rte_flow_error *error)
 697 {
 698         const struct rte_flow_action_of_set_vlan_vid *conf =
 699                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 700         int i = resource->actions_num;
 701         struct mlx5_modification_cmd *actions = resource->actions;
 702         struct field_modify_info *field = modify_vlan_out_first_vid;
 703
 704         if (i >= MLX5_MAX_MODIFY_NUM)
 705                 return rte_flow_error_set(error, EINVAL,
 706                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 707                          "too many items to modify");
 708         actions[i] = (struct mlx5_modification_cmd) {
 709                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 710                 .field = field->id,
 711                 .length = field->size,
 712                 .offset = field->offset,
 713         };
 714         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 715         actions[i].data1 = conf->vlan_vid;
 716         actions[i].data1 = actions[i].data1 << 16;
 717         resource->actions_num = ++i;
 718         return 0;
 719 }
 720
 721 /**
 722  * Convert modify-header set TP action to DV specification.
 723  *
 724  * @param[in,out] resource
 725  *   Pointer to the modify-header resource.
 726  * @param[in] action
 727  *   Pointer to action specification.
 728  * @param[in] items
 729  *   Pointer to rte_flow_item objects list.
 730  * @param[in] attr
 731  *   Pointer to flow attributes structure.
 732  * @param[in] dev_flow
 733  *   Pointer to the sub flow.
 734  * @param[in] tunnel_decap
 735  *   Whether action is after tunnel decapsulation.
 736  * @param[out] error
 737  *   Pointer to the error structure.
 738  *
 739  * @return
 740  *   0 on success, a negative errno value otherwise and rte_errno is set.
 741  */
 742 static int
 743 flow_dv_convert_action_modify_tp
 744                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 745                          const struct rte_flow_action *action,
 746                          const struct rte_flow_item *items,
 747                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 748                          bool tunnel_decap, struct rte_flow_error *error)
 749 {
 750         const struct rte_flow_action_set_tp *conf =
 751                 (const struct rte_flow_action_set_tp *)(action->conf);
 752         struct rte_flow_item item;
 753         struct rte_flow_item_udp udp;
 754         struct rte_flow_item_udp udp_mask;
 755         struct rte_flow_item_tcp tcp;
 756         struct rte_flow_item_tcp tcp_mask;
 757         struct field_modify_info *field;
 758
 759         if (!attr->valid)
 760                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 761         if (attr->udp) {
 762                 memset(&udp, 0, sizeof(udp));
 763                 memset(&udp_mask, 0, sizeof(udp_mask));
 764                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 765                         udp.hdr.src_port = conf->port;
 766                         udp_mask.hdr.src_port =
 767                                         rte_flow_item_udp_mask.hdr.src_port;
 768                 } else {
 769                         udp.hdr.dst_port = conf->port;
 770                         udp_mask.hdr.dst_port =
 771                                         rte_flow_item_udp_mask.hdr.dst_port;
 772                 }
 773                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 774                 item.spec = &udp;
 775                 item.mask = &udp_mask;
 776                 field = modify_udp;
 777         } else {
 778                 MLX5_ASSERT(attr->tcp);
 779                 memset(&tcp, 0, sizeof(tcp));
 780                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 781                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 782                         tcp.hdr.src_port = conf->port;
 783                         tcp_mask.hdr.src_port =
 784                                         rte_flow_item_tcp_mask.hdr.src_port;
 785                 } else {
 786                         tcp.hdr.dst_port = conf->port;
 787                         tcp_mask.hdr.dst_port =
 788                                         rte_flow_item_tcp_mask.hdr.dst_port;
 789                 }
 790                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 791                 item.spec = &tcp;
 792                 item.mask = &tcp_mask;
 793                 field = modify_tcp;
 794         }
 795         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 796                                              MLX5_MODIFICATION_TYPE_SET, error);
 797 }
 798
 799 /**
 800  * Convert modify-header set TTL action to DV specification.
 801  *
 802  * @param[in,out] resource
 803  *   Pointer to the modify-header resource.
 804  * @param[in] action
 805  *   Pointer to action specification.
 806  * @param[in] items
 807  *   Pointer to rte_flow_item objects list.
 808  * @param[in] attr
 809  *   Pointer to flow attributes structure.
 810  * @param[in] dev_flow
 811  *   Pointer to the sub flow.
 812  * @param[in] tunnel_decap
 813  *   Whether action is after tunnel decapsulation.
 814  * @param[out] error
 815  *   Pointer to the error structure.
 816  *
 817  * @return
 818  *   0 on success, a negative errno value otherwise and rte_errno is set.
 819  */
 820 static int
 821 flow_dv_convert_action_modify_ttl
 822                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 823                          const struct rte_flow_action *action,
 824                          const struct rte_flow_item *items,
 825                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 826                          bool tunnel_decap, struct rte_flow_error *error)
 827 {
 828         const struct rte_flow_action_set_ttl *conf =
 829                 (const struct rte_flow_action_set_ttl *)(action->conf);
 830         struct rte_flow_item item;
 831         struct rte_flow_item_ipv4 ipv4;
 832         struct rte_flow_item_ipv4 ipv4_mask;
 833         struct rte_flow_item_ipv6 ipv6;
 834         struct rte_flow_item_ipv6 ipv6_mask;
 835         struct field_modify_info *field;
 836
 837         if (!attr->valid)
 838                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 839         if (attr->ipv4) {
 840                 memset(&ipv4, 0, sizeof(ipv4));
 841                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 842                 ipv4.hdr.time_to_live = conf->ttl_value;
 843                 ipv4_mask.hdr.time_to_live = 0xFF;
 844                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 845                 item.spec = &ipv4;
 846                 item.mask = &ipv4_mask;
 847                 field = modify_ipv4;
 848         } else {
 849                 MLX5_ASSERT(attr->ipv6);
 850                 memset(&ipv6, 0, sizeof(ipv6));
 851                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 852                 ipv6.hdr.hop_limits = conf->ttl_value;
 853                 ipv6_mask.hdr.hop_limits = 0xFF;
 854                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 855                 item.spec = &ipv6;
 856                 item.mask = &ipv6_mask;
 857                 field = modify_ipv6;
 858         }
 859         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 860                                              MLX5_MODIFICATION_TYPE_SET, error);
 861 }
 862
 863 /**
 864  * Convert modify-header decrement TTL action to DV specification.
 865  *
 866  * @param[in,out] resource
 867  *   Pointer to the modify-header resource.
 868  * @param[in] action
 869  *   Pointer to action specification.
 870  * @param[in] items
 871  *   Pointer to rte_flow_item objects list.
 872  * @param[in] attr
 873  *   Pointer to flow attributes structure.
 874  * @param[in] dev_flow
 875  *   Pointer to the sub flow.
 876  * @param[in] tunnel_decap
 877  *   Whether action is after tunnel decapsulation.
 878  * @param[out] error
 879  *   Pointer to the error structure.
 880  *
 881  * @return
 882  *   0 on success, a negative errno value otherwise and rte_errno is set.
 883  */
 884 static int
 885 flow_dv_convert_action_modify_dec_ttl
 886                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 887                          const struct rte_flow_item *items,
 888                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 889                          bool tunnel_decap, struct rte_flow_error *error)
 890 {
 891         struct rte_flow_item item;
 892         struct rte_flow_item_ipv4 ipv4;
 893         struct rte_flow_item_ipv4 ipv4_mask;
 894         struct rte_flow_item_ipv6 ipv6;
 895         struct rte_flow_item_ipv6 ipv6_mask;
 896         struct field_modify_info *field;
 897
 898         if (!attr->valid)
 899                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 900         if (attr->ipv4) {
 901                 memset(&ipv4, 0, sizeof(ipv4));
 902                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 903                 ipv4.hdr.time_to_live = 0xFF;
 904                 ipv4_mask.hdr.time_to_live = 0xFF;
 905                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 906                 item.spec = &ipv4;
 907                 item.mask = &ipv4_mask;
 908                 field = modify_ipv4;
 909         } else {
 910                 MLX5_ASSERT(attr->ipv6);
 911                 memset(&ipv6, 0, sizeof(ipv6));
 912                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 913                 ipv6.hdr.hop_limits = 0xFF;
 914                 ipv6_mask.hdr.hop_limits = 0xFF;
 915                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 916                 item.spec = &ipv6;
 917                 item.mask = &ipv6_mask;
 918                 field = modify_ipv6;
 919         }
 920         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 921                                              MLX5_MODIFICATION_TYPE_ADD, error);
 922 }
 923
 924 /**
 925  * Convert modify-header increment/decrement TCP Sequence number
 926  * to DV specification.
 927  *
 928  * @param[in,out] resource
 929  *   Pointer to the modify-header resource.
 930  * @param[in] action
 931  *   Pointer to action specification.
 932  * @param[out] error
 933  *   Pointer to the error structure.
 934  *
 935  * @return
 936  *   0 on success, a negative errno value otherwise and rte_errno is set.
 937  */
 938 static int
 939 flow_dv_convert_action_modify_tcp_seq
 940                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 941                          const struct rte_flow_action *action,
 942                          struct rte_flow_error *error)
 943 {
 944         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 945         uint64_t value = rte_be_to_cpu_32(*conf);
 946         struct rte_flow_item item;
 947         struct rte_flow_item_tcp tcp;
 948         struct rte_flow_item_tcp tcp_mask;
 949
 950         memset(&tcp, 0, sizeof(tcp));
 951         memset(&tcp_mask, 0, sizeof(tcp_mask));
 952         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 953                 /*
 954                  * The HW has no decrement operation, only increment operation.
 955                  * To simulate decrement X from Y using increment operation
 956                  * we need to add UINT32_MAX X times to Y.
 957                  * Each adding of UINT32_MAX decrements Y by 1.
 958                  */
 959                 value *= UINT32_MAX;
 960         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 961         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 962         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 963         item.spec = &tcp;
 964         item.mask = &tcp_mask;
 965         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 966                                              MLX5_MODIFICATION_TYPE_ADD, error);
 967 }
 968
 969 /**
 970  * Convert modify-header increment/decrement TCP Acknowledgment number
 971  * to DV specification.
 972  *
 973  * @param[in,out] resource
 974  *   Pointer to the modify-header resource.
 975  * @param[in] action
 976  *   Pointer to action specification.
 977  * @param[out] error
 978  *   Pointer to the error structure.
 979  *
 980  * @return
 981  *   0 on success, a negative errno value otherwise and rte_errno is set.
 982  */
 983 static int
 984 flow_dv_convert_action_modify_tcp_ack
 985                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 986                          const struct rte_flow_action *action,
 987                          struct rte_flow_error *error)
 988 {
 989         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 990         uint64_t value = rte_be_to_cpu_32(*conf);
 991         struct rte_flow_item item;
 992         struct rte_flow_item_tcp tcp;
 993         struct rte_flow_item_tcp tcp_mask;
 994
 995         memset(&tcp, 0, sizeof(tcp));
 996         memset(&tcp_mask, 0, sizeof(tcp_mask));
 997         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 998                 /*
 999                  * The HW has no decrement operation, only increment operation.
1000                  * To simulate decrement X from Y using increment operation
1001                  * we need to add UINT32_MAX X times to Y.
1002                  * Each adding of UINT32_MAX decrements Y by 1.
1003                  */
1004                 value *= UINT32_MAX;
1005         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
1006         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
1007         item.type = RTE_FLOW_ITEM_TYPE_TCP;
1008         item.spec = &tcp;
1009         item.mask = &tcp_mask;
1010         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
1011                                              MLX5_MODIFICATION_TYPE_ADD, error);
1012 }
1013
1014 static enum mlx5_modification_field reg_to_field[] = {
1015         [REG_NON] = MLX5_MODI_OUT_NONE,
1016         [REG_A] = MLX5_MODI_META_DATA_REG_A,
1017         [REG_B] = MLX5_MODI_META_DATA_REG_B,
1018         [REG_C_0] = MLX5_MODI_META_REG_C_0,
1019         [REG_C_1] = MLX5_MODI_META_REG_C_1,
1020         [REG_C_2] = MLX5_MODI_META_REG_C_2,
1021         [REG_C_3] = MLX5_MODI_META_REG_C_3,
1022         [REG_C_4] = MLX5_MODI_META_REG_C_4,
1023         [REG_C_5] = MLX5_MODI_META_REG_C_5,
1024         [REG_C_6] = MLX5_MODI_META_REG_C_6,
1025         [REG_C_7] = MLX5_MODI_META_REG_C_7,
1026 };
1027
1028 /**
1029  * Convert register set to DV specification.
1030  *
1031  * @param[in,out] resource
1032  *   Pointer to the modify-header resource.
1033  * @param[in] action
1034  *   Pointer to action specification.
1035  * @param[out] error
1036  *   Pointer to the error structure.
1037  *
1038  * @return
1039  *   0 on success, a negative errno value otherwise and rte_errno is set.
1040  */
1041 static int
1042 flow_dv_convert_action_set_reg
1043                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1044                          const struct rte_flow_action *action,
1045                          struct rte_flow_error *error)
1046 {
1047         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
1048         struct mlx5_modification_cmd *actions = resource->actions;
1049         uint32_t i = resource->actions_num;
1050
1051         if (i >= MLX5_MAX_MODIFY_NUM)
1052                 return rte_flow_error_set(error, EINVAL,
1053                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1054                                           "too many items to modify");
1055         MLX5_ASSERT(conf->id != REG_NON);
1056         MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
1057         actions[i] = (struct mlx5_modification_cmd) {
1058                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1059                 .field = reg_to_field[conf->id],
1060                 .offset = conf->offset,
1061                 .length = conf->length,
1062         };
1063         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1064         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1065         ++i;
1066         resource->actions_num = i;
1067         return 0;
1068 }
1069
1070 /**
1071  * Convert SET_TAG action to DV specification.
1072  *
1073  * @param[in] dev
1074  *   Pointer to the rte_eth_dev structure.
1075  * @param[in,out] resource
1076  *   Pointer to the modify-header resource.
1077  * @param[in] conf
1078  *   Pointer to action specification.
1079  * @param[out] error
1080  *   Pointer to the error structure.
1081  *
1082  * @return
1083  *   0 on success, a negative errno value otherwise and rte_errno is set.
1084  */
1085 static int
1086 flow_dv_convert_action_set_tag
1087                         (struct rte_eth_dev *dev,
1088                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1089                          const struct rte_flow_action_set_tag *conf,
1090                          struct rte_flow_error *error)
1091 {
1092         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1093         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1094         struct rte_flow_item item = {
1095                 .spec = &data,
1096                 .mask = &mask,
1097         };
1098         struct field_modify_info reg_c_x[] = {
1099                 [1] = {0, 0, 0},
1100         };
1101         enum mlx5_modification_field reg_type;
1102         int ret;
1103
1104         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1105         if (ret < 0)
1106                 return ret;
1107         MLX5_ASSERT(ret != REG_NON);
1108         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1109         reg_type = reg_to_field[ret];
1110         MLX5_ASSERT(reg_type > 0);
1111         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1112         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1113                                              MLX5_MODIFICATION_TYPE_SET, error);
1114 }
1115
1116 /**
1117  * Convert internal COPY_REG action to DV specification.
1118  *
1119  * @param[in] dev
1120  *   Pointer to the rte_eth_dev structure.
1121  * @param[in,out] res
1122  *   Pointer to the modify-header resource.
1123  * @param[in] action
1124  *   Pointer to action specification.
1125  * @param[out] error
1126  *   Pointer to the error structure.
1127  *
1128  * @return
1129  *   0 on success, a negative errno value otherwise and rte_errno is set.
1130  */
1131 static int
1132 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1133                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1134                                  const struct rte_flow_action *action,
1135                                  struct rte_flow_error *error)
1136 {
1137         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1138         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1139         struct rte_flow_item item = {
1140                 .spec = NULL,
1141                 .mask = &mask,
1142         };
1143         struct field_modify_info reg_src[] = {
1144                 {4, 0, reg_to_field[conf->src]},
1145                 {0, 0, 0},
1146         };
1147         struct field_modify_info reg_dst = {
1148                 .offset = 0,
1149                 .id = reg_to_field[conf->dst],
1150         };
1151         /* Adjust reg_c[0] usage according to reported mask. */
1152         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1153                 struct mlx5_priv *priv = dev->data->dev_private;
1154                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1155
1156                 MLX5_ASSERT(reg_c0);
1157                 MLX5_ASSERT(priv->sh->config.dv_xmeta_en !=
1158                             MLX5_XMETA_MODE_LEGACY);
1159                 if (conf->dst == REG_C_0) {
1160                         /* Copy to reg_c[0], within mask only. */
1161                         reg_dst.offset = rte_bsf32(reg_c0);
1162                         mask = rte_cpu_to_be_32(reg_c0 >> reg_dst.offset);
1163                 } else {
1164                         reg_dst.offset = 0;
1165                         mask = rte_cpu_to_be_32(reg_c0);
1166                 }
1167         }
1168         return flow_dv_convert_modify_action(&item,
1169                                              reg_src, &reg_dst, res,
1170                                              MLX5_MODIFICATION_TYPE_COPY,
1171                                              error);
1172 }
1173
1174 /**
1175  * Convert MARK action to DV specification. This routine is used
1176  * in extensive metadata only and requires metadata register to be
1177  * handled. In legacy mode hardware tag resource is engaged.
1178  *
1179  * @param[in] dev
1180  *   Pointer to the rte_eth_dev structure.
1181  * @param[in] conf
1182  *   Pointer to MARK action specification.
1183  * @param[in,out] resource
1184  *   Pointer to the modify-header resource.
1185  * @param[out] error
1186  *   Pointer to the error structure.
1187  *
1188  * @return
1189  *   0 on success, a negative errno value otherwise and rte_errno is set.
1190  */
1191 static int
1192 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1193                             const struct rte_flow_action_mark *conf,
1194                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1195                             struct rte_flow_error *error)
1196 {
1197         struct mlx5_priv *priv = dev->data->dev_private;
1198         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1199                                            priv->sh->dv_mark_mask);
1200         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1201         struct rte_flow_item item = {
1202                 .spec = &data,
1203                 .mask = &mask,
1204         };
1205         struct field_modify_info reg_c_x[] = {
1206                 [1] = {0, 0, 0},
1207         };
1208         int reg;
1209
1210         if (!mask)
1211                 return rte_flow_error_set(error, EINVAL,
1212                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1213                                           NULL, "zero mark action mask");
1214         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1215         if (reg < 0)
1216                 return reg;
1217         MLX5_ASSERT(reg > 0);
1218         if (reg == REG_C_0) {
1219                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1220                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1221
1222                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1223                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1224                 mask = rte_cpu_to_be_32(mask << shl_c0);
1225         }
1226         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1227         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1228                                              MLX5_MODIFICATION_TYPE_SET, error);
1229 }
1230
1231 /**
1232  * Get metadata register index for specified steering domain.
1233  *
1234  * @param[in] dev
1235  *   Pointer to the rte_eth_dev structure.
1236  * @param[in] attr
1237  *   Attributes of flow to determine steering domain.
1238  * @param[out] error
1239  *   Pointer to the error structure.
1240  *
1241  * @return
1242  *   positive index on success, a negative errno value otherwise
1243  *   and rte_errno is set.
1244  */
1245 static enum modify_reg
1246 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1247                          const struct rte_flow_attr *attr,
1248                          struct rte_flow_error *error)
1249 {
1250         int reg =
1251                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1252                                           MLX5_METADATA_FDB :
1253                                             attr->egress ?
1254                                             MLX5_METADATA_TX :
1255                                             MLX5_METADATA_RX, 0, error);
1256         if (reg < 0)
1257                 return rte_flow_error_set(error,
1258                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1259                                           NULL, "unavailable "
1260                                           "metadata register");
1261         return reg;
1262 }
1263
1264 /**
1265  * Convert SET_META action to DV specification.
1266  *
1267  * @param[in] dev
1268  *   Pointer to the rte_eth_dev structure.
1269  * @param[in,out] resource
1270  *   Pointer to the modify-header resource.
1271  * @param[in] attr
1272  *   Attributes of flow that includes this item.
1273  * @param[in] conf
1274  *   Pointer to action specification.
1275  * @param[out] error
1276  *   Pointer to the error structure.
1277  *
1278  * @return
1279  *   0 on success, a negative errno value otherwise and rte_errno is set.
1280  */
1281 static int
1282 flow_dv_convert_action_set_meta
1283                         (struct rte_eth_dev *dev,
1284                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1285                          const struct rte_flow_attr *attr,
1286                          const struct rte_flow_action_set_meta *conf,
1287                          struct rte_flow_error *error)
1288 {
1289         uint32_t mask = rte_cpu_to_be_32(conf->mask);
1290         uint32_t data = rte_cpu_to_be_32(conf->data) & mask;
1291         struct rte_flow_item item = {
1292                 .spec = &data,
1293                 .mask = &mask,
1294         };
1295         struct field_modify_info reg_c_x[] = {
1296                 [1] = {0, 0, 0},
1297         };
1298         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1299
1300         if (reg < 0)
1301                 return reg;
1302         MLX5_ASSERT(reg != REG_NON);
1303         if (reg == REG_C_0) {
1304                 struct mlx5_priv *priv = dev->data->dev_private;
1305                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1306                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1307
1308                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1309                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1310                 mask = rte_cpu_to_be_32(mask << shl_c0);
1311         }
1312         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1313         /* The routine expects parameters in memory as big-endian ones. */
1314         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1315                                              MLX5_MODIFICATION_TYPE_SET, error);
1316 }
1317
1318 /**
1319  * Convert modify-header set IPv4 DSCP action to DV specification.
1320  *
1321  * @param[in,out] resource
1322  *   Pointer to the modify-header resource.
1323  * @param[in] action
1324  *   Pointer to action specification.
1325  * @param[out] error
1326  *   Pointer to the error structure.
1327  *
1328  * @return
1329  *   0 on success, a negative errno value otherwise and rte_errno is set.
1330  */
1331 static int
1332 flow_dv_convert_action_modify_ipv4_dscp
1333                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1334                          const struct rte_flow_action *action,
1335                          struct rte_flow_error *error)
1336 {
1337         const struct rte_flow_action_set_dscp *conf =
1338                 (const struct rte_flow_action_set_dscp *)(action->conf);
1339         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1340         struct rte_flow_item_ipv4 ipv4;
1341         struct rte_flow_item_ipv4 ipv4_mask;
1342
1343         memset(&ipv4, 0, sizeof(ipv4));
1344         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1345         ipv4.hdr.type_of_service = conf->dscp;
1346         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1347         item.spec = &ipv4;
1348         item.mask = &ipv4_mask;
1349         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1350                                              MLX5_MODIFICATION_TYPE_SET, error);
1351 }
1352
1353 /**
1354  * Convert modify-header set IPv6 DSCP action to DV specification.
1355  *
1356  * @param[in,out] resource
1357  *   Pointer to the modify-header resource.
1358  * @param[in] action
1359  *   Pointer to action specification.
1360  * @param[out] error
1361  *   Pointer to the error structure.
1362  *
1363  * @return
1364  *   0 on success, a negative errno value otherwise and rte_errno is set.
1365  */
1366 static int
1367 flow_dv_convert_action_modify_ipv6_dscp
1368                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1369                          const struct rte_flow_action *action,
1370                          struct rte_flow_error *error)
1371 {
1372         const struct rte_flow_action_set_dscp *conf =
1373                 (const struct rte_flow_action_set_dscp *)(action->conf);
1374         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1375         struct rte_flow_item_ipv6 ipv6;
1376         struct rte_flow_item_ipv6 ipv6_mask;
1377
1378         memset(&ipv6, 0, sizeof(ipv6));
1379         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1380         /*
1381          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1382          * rdma-core only accept the DSCP bits byte aligned start from
1383          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1384          * bits in IPv6 case as rdma-core requires byte aligned value.
1385          */
1386         ipv6.hdr.vtc_flow = conf->dscp;
1387         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1388         item.spec = &ipv6;
1389         item.mask = &ipv6_mask;
1390         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1391                                              MLX5_MODIFICATION_TYPE_SET, error);
1392 }
1393
1394 static int
1395 mlx5_flow_item_field_width(struct rte_eth_dev *dev,
1396                            enum rte_flow_field_id field, int inherit,
1397                            const struct rte_flow_attr *attr,
1398                            struct rte_flow_error *error)
1399 {
1400         struct mlx5_priv *priv = dev->data->dev_private;
1401
1402         switch (field) {
1403         case RTE_FLOW_FIELD_START:
1404                 return 32;
1405         case RTE_FLOW_FIELD_MAC_DST:
1406         case RTE_FLOW_FIELD_MAC_SRC:
1407                 return 48;
1408         case RTE_FLOW_FIELD_VLAN_TYPE:
1409                 return 16;
1410         case RTE_FLOW_FIELD_VLAN_ID:
1411                 return 12;
1412         case RTE_FLOW_FIELD_MAC_TYPE:
1413                 return 16;
1414         case RTE_FLOW_FIELD_IPV4_DSCP:
1415                 return 6;
1416         case RTE_FLOW_FIELD_IPV4_TTL:
1417                 return 8;
1418         case RTE_FLOW_FIELD_IPV4_SRC:
1419         case RTE_FLOW_FIELD_IPV4_DST:
1420                 return 32;
1421         case RTE_FLOW_FIELD_IPV6_DSCP:
1422                 return 6;
1423         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1424                 return 8;
1425         case RTE_FLOW_FIELD_IPV6_SRC:
1426         case RTE_FLOW_FIELD_IPV6_DST:
1427                 return 128;
1428         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1429         case RTE_FLOW_FIELD_TCP_PORT_DST:
1430                 return 16;
1431         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1432         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1433                 return 32;
1434         case RTE_FLOW_FIELD_TCP_FLAGS:
1435                 return 9;
1436         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1437         case RTE_FLOW_FIELD_UDP_PORT_DST:
1438                 return 16;
1439         case RTE_FLOW_FIELD_VXLAN_VNI:
1440         case RTE_FLOW_FIELD_GENEVE_VNI:
1441                 return 24;
1442         case RTE_FLOW_FIELD_GTP_TEID:
1443         case RTE_FLOW_FIELD_TAG:
1444                 return 32;
1445         case RTE_FLOW_FIELD_MARK:
1446                 return __builtin_popcount(priv->sh->dv_mark_mask);
1447         case RTE_FLOW_FIELD_META:
1448                 return (flow_dv_get_metadata_reg(dev, attr, error) == REG_C_0) ?
1449                         __builtin_popcount(priv->sh->dv_meta_mask) : 32;
1450         case RTE_FLOW_FIELD_POINTER:
1451         case RTE_FLOW_FIELD_VALUE:
1452                 return inherit < 0 ? 0 : inherit;
1453         default:
1454                 MLX5_ASSERT(false);
1455         }
1456         return 0;
1457 }
1458
1459 static void
1460 mlx5_flow_field_id_to_modify_info
1461                 (const struct rte_flow_action_modify_data *data,
1462                  struct field_modify_info *info, uint32_t *mask,
1463                  uint32_t width, struct rte_eth_dev *dev,
1464                  const struct rte_flow_attr *attr, struct rte_flow_error *error)
1465 {
1466         struct mlx5_priv *priv = dev->data->dev_private;
1467         uint32_t idx = 0;
1468         uint32_t off = 0;
1469
1470         switch (data->field) {
1471         case RTE_FLOW_FIELD_START:
1472                 /* not supported yet */
1473                 MLX5_ASSERT(false);
1474                 break;
1475         case RTE_FLOW_FIELD_MAC_DST:
1476                 off = data->offset > 16 ? data->offset - 16 : 0;
1477                 if (mask) {
1478                         if (data->offset < 16) {
1479                                 info[idx] = (struct field_modify_info){2, 4,
1480                                                 MLX5_MODI_OUT_DMAC_15_0};
1481                                 if (width < 16) {
1482                                         mask[1] = rte_cpu_to_be_16(0xffff >>
1483                                                                  (16 - width));
1484                                         width = 0;
1485                                 } else {
1486                                         mask[1] = RTE_BE16(0xffff);
1487                                         width -= 16;
1488                                 }
1489                                 if (!width)
1490                                         break;
1491                                 ++idx;
1492                         }
1493                         info[idx] = (struct field_modify_info){4, 0,
1494                                                 MLX5_MODI_OUT_DMAC_47_16};
1495                         mask[0] = rte_cpu_to_be_32((0xffffffff >>
1496                                                     (32 - width)) << off);
1497                 } else {
1498                         if (data->offset < 16)
1499                                 info[idx++] = (struct field_modify_info){2, 0,
1500                                                 MLX5_MODI_OUT_DMAC_15_0};
1501                         info[idx] = (struct field_modify_info){4, off,
1502                                                 MLX5_MODI_OUT_DMAC_47_16};
1503                 }
1504                 break;
1505         case RTE_FLOW_FIELD_MAC_SRC:
1506                 off = data->offset > 16 ? data->offset - 16 : 0;
1507                 if (mask) {
1508                         if (data->offset < 16) {
1509                                 info[idx] = (struct field_modify_info){2, 4,
1510                                                 MLX5_MODI_OUT_SMAC_15_0};
1511                                 if (width < 16) {
1512                                         mask[1] = rte_cpu_to_be_16(0xffff >>
1513                                                                  (16 - width));
1514                                         width = 0;
1515                                 } else {
1516                                         mask[1] = RTE_BE16(0xffff);
1517                                         width -= 16;
1518                                 }
1519                                 if (!width)
1520                                         break;
1521                                 ++idx;
1522                         }
1523                         info[idx] = (struct field_modify_info){4, 0,
1524                                                 MLX5_MODI_OUT_SMAC_47_16};
1525                         mask[0] = rte_cpu_to_be_32((0xffffffff >>
1526                                                     (32 - width)) << off);
1527                 } else {
1528                         if (data->offset < 16)
1529                                 info[idx++] = (struct field_modify_info){2, 0,
1530                                                 MLX5_MODI_OUT_SMAC_15_0};
1531                         info[idx] = (struct field_modify_info){4, off,
1532                                                 MLX5_MODI_OUT_SMAC_47_16};
1533                 }
1534                 break;
1535         case RTE_FLOW_FIELD_VLAN_TYPE:
1536                 /* not supported yet */
1537                 break;
1538         case RTE_FLOW_FIELD_VLAN_ID:
1539                 info[idx] = (struct field_modify_info){2, 0,
1540                                         MLX5_MODI_OUT_FIRST_VID};
1541                 if (mask)
1542                         mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
1543                 break;
1544         case RTE_FLOW_FIELD_MAC_TYPE:
1545                 info[idx] = (struct field_modify_info){2, 0,
1546                                         MLX5_MODI_OUT_ETHERTYPE};
1547                 if (mask)
1548                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1549                 break;
1550         case RTE_FLOW_FIELD_IPV4_DSCP:
1551                 info[idx] = (struct field_modify_info){1, 0,
1552                                         MLX5_MODI_OUT_IP_DSCP};
1553                 if (mask)
1554                         mask[idx] = 0x3f >> (6 - width);
1555                 break;
1556         case RTE_FLOW_FIELD_IPV4_TTL:
1557                 info[idx] = (struct field_modify_info){1, 0,
1558                                         MLX5_MODI_OUT_IPV4_TTL};
1559                 if (mask)
1560                         mask[idx] = 0xff >> (8 - width);
1561                 break;
1562         case RTE_FLOW_FIELD_IPV4_SRC:
1563                 info[idx] = (struct field_modify_info){4, 0,
1564                                         MLX5_MODI_OUT_SIPV4};
1565                 if (mask)
1566                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1567                                                      (32 - width));
1568                 break;
1569         case RTE_FLOW_FIELD_IPV4_DST:
1570                 info[idx] = (struct field_modify_info){4, 0,
1571                                         MLX5_MODI_OUT_DIPV4};
1572                 if (mask)
1573                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1574                                                      (32 - width));
1575                 break;
1576         case RTE_FLOW_FIELD_IPV6_DSCP:
1577                 info[idx] = (struct field_modify_info){1, 0,
1578                                         MLX5_MODI_OUT_IP_DSCP};
1579                 if (mask)
1580                         mask[idx] = 0x3f >> (6 - width);
1581                 break;
1582         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1583                 info[idx] = (struct field_modify_info){1, 0,
1584                                         MLX5_MODI_OUT_IPV6_HOPLIMIT};
1585                 if (mask)
1586                         mask[idx] = 0xff >> (8 - width);
1587                 break;
1588         case RTE_FLOW_FIELD_IPV6_SRC:
1589                 if (mask) {
1590                         if (data->offset < 32) {
1591                                 info[idx] = (struct field_modify_info){4, 12,
1592                                                 MLX5_MODI_OUT_SIPV6_31_0};
1593                                 if (width < 32) {
1594                                         mask[3] =
1595                                                 rte_cpu_to_be_32(0xffffffff >>
1596                                                                  (32 - width));
1597                                         width = 0;
1598                                 } else {
1599                                         mask[3] = RTE_BE32(0xffffffff);
1600                                         width -= 32;
1601                                 }
1602                                 if (!width)
1603                                         break;
1604                                 ++idx;
1605                         }
1606                         if (data->offset < 64) {
1607                                 info[idx] = (struct field_modify_info){4, 8,
1608                                                 MLX5_MODI_OUT_SIPV6_63_32};
1609                                 if (width < 32) {
1610                                         mask[2] =
1611                                                 rte_cpu_to_be_32(0xffffffff >>
1612                                                                  (32 - width));
1613                                         width = 0;
1614                                 } else {
1615                                         mask[2] = RTE_BE32(0xffffffff);
1616                                         width -= 32;
1617                                 }
1618                                 if (!width)
1619                                         break;
1620                                 ++idx;
1621                         }
1622                         if (data->offset < 96) {
1623                                 info[idx] = (struct field_modify_info){4, 4,
1624                                                 MLX5_MODI_OUT_SIPV6_95_64};
1625                                 if (width < 32) {
1626                                         mask[1] =
1627                                                 rte_cpu_to_be_32(0xffffffff >>
1628                                                                  (32 - width));
1629                                         width = 0;
1630                                 } else {
1631                                         mask[1] = RTE_BE32(0xffffffff);
1632                                         width -= 32;
1633                                 }
1634                                 if (!width)
1635                                         break;
1636                                 ++idx;
1637                         }
1638                         info[idx] = (struct field_modify_info){4, 0,
1639                                                 MLX5_MODI_OUT_SIPV6_127_96};
1640                         mask[0] = rte_cpu_to_be_32(0xffffffff >> (32 - width));
1641                 } else {
1642                         if (data->offset < 32)
1643                                 info[idx++] = (struct field_modify_info){4, 0,
1644                                                 MLX5_MODI_OUT_SIPV6_31_0};
1645                         if (data->offset < 64)
1646                                 info[idx++] = (struct field_modify_info){4, 0,
1647                                                 MLX5_MODI_OUT_SIPV6_63_32};
1648                         if (data->offset < 96)
1649                                 info[idx++] = (struct field_modify_info){4, 0,
1650                                                 MLX5_MODI_OUT_SIPV6_95_64};
1651                         if (data->offset < 128)
1652                                 info[idx++] = (struct field_modify_info){4, 0,
1653                                                 MLX5_MODI_OUT_SIPV6_127_96};
1654                 }
1655                 break;
1656         case RTE_FLOW_FIELD_IPV6_DST:
1657                 if (mask) {
1658                         if (data->offset < 32) {
1659                                 info[idx] = (struct field_modify_info){4, 12,
1660                                                 MLX5_MODI_OUT_DIPV6_31_0};
1661                                 if (width < 32) {
1662                                         mask[3] =
1663                                                 rte_cpu_to_be_32(0xffffffff >>
1664                                                                  (32 - width));
1665                                         width = 0;
1666                                 } else {
1667                                         mask[3] = RTE_BE32(0xffffffff);
1668                                         width -= 32;
1669                                 }
1670                                 if (!width)
1671                                         break;
1672                                 ++idx;
1673                         }
1674                         if (data->offset < 64) {
1675                                 info[idx] = (struct field_modify_info){4, 8,
1676                                                 MLX5_MODI_OUT_DIPV6_63_32};
1677                                 if (width < 32) {
1678                                         mask[2] =
1679                                                 rte_cpu_to_be_32(0xffffffff >>
1680                                                                  (32 - width));
1681                                         width = 0;
1682                                 } else {
1683                                         mask[2] = RTE_BE32(0xffffffff);
1684                                         width -= 32;
1685                                 }
1686                                 if (!width)
1687                                         break;
1688                                 ++idx;
1689                         }
1690                         if (data->offset < 96) {
1691                                 info[idx] = (struct field_modify_info){4, 4,
1692                                                 MLX5_MODI_OUT_DIPV6_95_64};
1693                                 if (width < 32) {
1694                                         mask[1] =
1695                                                 rte_cpu_to_be_32(0xffffffff >>
1696                                                                  (32 - width));
1697                                         width = 0;
1698                                 } else {
1699                                         mask[1] = RTE_BE32(0xffffffff);
1700                                         width -= 32;
1701                                 }
1702                                 if (!width)
1703                                         break;
1704                                 ++idx;
1705                         }
1706                         info[idx] = (struct field_modify_info){4, 0,
1707                                                 MLX5_MODI_OUT_DIPV6_127_96};
1708                         mask[0] = rte_cpu_to_be_32(0xffffffff >> (32 - width));
1709                 } else {
1710                         if (data->offset < 32)
1711                                 info[idx++] = (struct field_modify_info){4, 0,
1712                                                 MLX5_MODI_OUT_DIPV6_31_0};
1713                         if (data->offset < 64)
1714                                 info[idx++] = (struct field_modify_info){4, 0,
1715                                                 MLX5_MODI_OUT_DIPV6_63_32};
1716                         if (data->offset < 96)
1717                                 info[idx++] = (struct field_modify_info){4, 0,
1718                                                 MLX5_MODI_OUT_DIPV6_95_64};
1719                         if (data->offset < 128)
1720                                 info[idx++] = (struct field_modify_info){4, 0,
1721                                                 MLX5_MODI_OUT_DIPV6_127_96};
1722                 }
1723                 break;
1724         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1725                 info[idx] = (struct field_modify_info){2, 0,
1726                                         MLX5_MODI_OUT_TCP_SPORT};
1727                 if (mask)
1728                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1729                 break;
1730         case RTE_FLOW_FIELD_TCP_PORT_DST:
1731                 info[idx] = (struct field_modify_info){2, 0,
1732                                         MLX5_MODI_OUT_TCP_DPORT};
1733                 if (mask)
1734                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1735                 break;
1736         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1737                 info[idx] = (struct field_modify_info){4, 0,
1738                                         MLX5_MODI_OUT_TCP_SEQ_NUM};
1739                 if (mask)
1740                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1741                                                      (32 - width));
1742                 break;
1743         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1744                 info[idx] = (struct field_modify_info){4, 0,
1745                                         MLX5_MODI_OUT_TCP_ACK_NUM};
1746                 if (mask)
1747                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1748                                                      (32 - width));
1749                 break;
1750         case RTE_FLOW_FIELD_TCP_FLAGS:
1751                 info[idx] = (struct field_modify_info){2, 0,
1752                                         MLX5_MODI_OUT_TCP_FLAGS};
1753                 if (mask)
1754                         mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
1755                 break;
1756         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1757                 info[idx] = (struct field_modify_info){2, 0,
1758                                         MLX5_MODI_OUT_UDP_SPORT};
1759                 if (mask)
1760                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1761                 break;
1762         case RTE_FLOW_FIELD_UDP_PORT_DST:
1763                 info[idx] = (struct field_modify_info){2, 0,
1764                                         MLX5_MODI_OUT_UDP_DPORT};
1765                 if (mask)
1766                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1767                 break;
1768         case RTE_FLOW_FIELD_VXLAN_VNI:
1769                 /* not supported yet */
1770                 break;
1771         case RTE_FLOW_FIELD_GENEVE_VNI:
1772                 /* not supported yet*/
1773                 break;
1774         case RTE_FLOW_FIELD_GTP_TEID:
1775                 info[idx] = (struct field_modify_info){4, 0,
1776                                         MLX5_MODI_GTP_TEID};
1777                 if (mask)
1778                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1779                                                      (32 - width));
1780                 break;
1781         case RTE_FLOW_FIELD_TAG:
1782                 {
1783                         int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
1784                                                    data->level, error);
1785                         if (reg < 0)
1786                                 return;
1787                         MLX5_ASSERT(reg != REG_NON);
1788                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1789                         info[idx] = (struct field_modify_info){4, 0,
1790                                                 reg_to_field[reg]};
1791                         if (mask)
1792                                 mask[idx] =
1793                                         rte_cpu_to_be_32(0xffffffff >>
1794                                                          (32 - width));
1795                 }
1796                 break;
1797         case RTE_FLOW_FIELD_MARK:
1798                 {
1799                         uint32_t mark_mask = priv->sh->dv_mark_mask;
1800                         uint32_t mark_count = __builtin_popcount(mark_mask);
1801                         int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
1802                                                        0, error);
1803                         if (reg < 0)
1804                                 return;
1805                         MLX5_ASSERT(reg != REG_NON);
1806                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1807                         info[idx] = (struct field_modify_info){4, 0,
1808                                                 reg_to_field[reg]};
1809                         if (mask)
1810                                 mask[idx] = rte_cpu_to_be_32((mark_mask >>
1811                                          (mark_count - width)) & mark_mask);
1812                 }
1813                 break;
1814         case RTE_FLOW_FIELD_META:
1815                 {
1816                         uint32_t meta_mask = priv->sh->dv_meta_mask;
1817                         uint32_t meta_count = __builtin_popcount(meta_mask);
1818                         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1819                         if (reg < 0)
1820                                 return;
1821                         MLX5_ASSERT(reg != REG_NON);
1822                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1823                         info[idx] = (struct field_modify_info){4, 0,
1824                                                 reg_to_field[reg]};
1825                         if (mask)
1826                                 mask[idx] = rte_cpu_to_be_32((meta_mask >>
1827                                         (meta_count - width)) & meta_mask);
1828                 }
1829                 break;
1830         case RTE_FLOW_FIELD_POINTER:
1831         case RTE_FLOW_FIELD_VALUE:
1832         default:
1833                 MLX5_ASSERT(false);
1834                 break;
1835         }
1836 }
1837
1838 /**
1839  * Convert modify_field action to DV specification.
1840  *
1841  * @param[in] dev
1842  *   Pointer to the rte_eth_dev structure.
1843  * @param[in,out] resource
1844  *   Pointer to the modify-header resource.
1845  * @param[in] action
1846  *   Pointer to action specification.
1847  * @param[in] attr
1848  *   Attributes of flow that includes this item.
1849  * @param[out] error
1850  *   Pointer to the error structure.
1851  *
1852  * @return
1853  *   0 on success, a negative errno value otherwise and rte_errno is set.
1854  */
1855 static int
1856 flow_dv_convert_action_modify_field
1857                         (struct rte_eth_dev *dev,
1858                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1859                          const struct rte_flow_action *action,
1860                          const struct rte_flow_attr *attr,
1861                          struct rte_flow_error *error)
1862 {
1863         const struct rte_flow_action_modify_field *conf =
1864                 (const struct rte_flow_action_modify_field *)(action->conf);
1865         struct rte_flow_item item = {
1866                 .spec = NULL,
1867                 .mask = NULL
1868         };
1869         struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
1870                                                                 {0, 0, 0} };
1871         struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
1872                                                                 {0, 0, 0} };
1873         uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1874         uint32_t type, meta = 0;
1875
1876         if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
1877             conf->src.field == RTE_FLOW_FIELD_VALUE) {
1878                 type = MLX5_MODIFICATION_TYPE_SET;
1879                 /** For SET fill the destination field (field) first. */
1880                 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
1881                                                   conf->width, dev,
1882                                                   attr, error);
1883                 item.spec = conf->src.field == RTE_FLOW_FIELD_POINTER ?
1884                                         (void *)(uintptr_t)conf->src.pvalue :
1885                                         (void *)(uintptr_t)&conf->src.value;
1886                 if (conf->dst.field == RTE_FLOW_FIELD_META) {
1887                         meta = *(const unaligned_uint32_t *)item.spec;
1888                         meta = rte_cpu_to_be_32(meta);
1889                         item.spec = &meta;
1890                 }
1891         } else {
1892                 type = MLX5_MODIFICATION_TYPE_COPY;
1893                 /** For COPY fill the destination field (dcopy) without mask. */
1894                 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
1895                                                   conf->width, dev,
1896                                                   attr, error);
1897                 /** Then construct the source field (field) with mask. */
1898                 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
1899                                                   conf->width, dev,
1900                                                   attr, error);
1901         }
1902         item.mask = &mask;
1903         return flow_dv_convert_modify_action(&item,
1904                         field, dcopy, resource, type, error);
1905 }
1906
1907 /**
1908  * Validate MARK item.
1909  *
1910  * @param[in] dev
1911  *   Pointer to the rte_eth_dev structure.
1912  * @param[in] item
1913  *   Item specification.
1914  * @param[in] attr
1915  *   Attributes of flow that includes this item.
1916  * @param[out] error
1917  *   Pointer to error structure.
1918  *
1919  * @return
1920  *   0 on success, a negative errno value otherwise and rte_errno is set.
1921  */
1922 static int
1923 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1924                            const struct rte_flow_item *item,
1925                            const struct rte_flow_attr *attr __rte_unused,
1926                            struct rte_flow_error *error)
1927 {
1928         struct mlx5_priv *priv = dev->data->dev_private;
1929         struct mlx5_sh_config *config = &priv->sh->config;
1930         const struct rte_flow_item_mark *spec = item->spec;
1931         const struct rte_flow_item_mark *mask = item->mask;
1932         const struct rte_flow_item_mark nic_mask = {
1933                 .id = priv->sh->dv_mark_mask,
1934         };
1935         int ret;
1936
1937         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1938                 return rte_flow_error_set(error, ENOTSUP,
1939                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1940                                           "extended metadata feature"
1941                                           " isn't enabled");
1942         if (!mlx5_flow_ext_mreg_supported(dev))
1943                 return rte_flow_error_set(error, ENOTSUP,
1944                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1945                                           "extended metadata register"
1946                                           " isn't supported");
1947         if (!nic_mask.id)
1948                 return rte_flow_error_set(error, ENOTSUP,
1949                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1950                                           "extended metadata register"
1951                                           " isn't available");
1952         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1953         if (ret < 0)
1954                 return ret;
1955         if (!spec)
1956                 return rte_flow_error_set(error, EINVAL,
1957                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1958                                           item->spec,
1959                                           "data cannot be empty");
1960         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1961                 return rte_flow_error_set(error, EINVAL,
1962                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1963                                           &spec->id,
1964                                           "mark id exceeds the limit");
1965         if (!mask)
1966                 mask = &nic_mask;
1967         if (!mask->id)
1968                 return rte_flow_error_set(error, EINVAL,
1969                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1970                                         "mask cannot be zero");
1971
1972         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1973                                         (const uint8_t *)&nic_mask,
1974                                         sizeof(struct rte_flow_item_mark),
1975                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1976         if (ret < 0)
1977                 return ret;
1978         return 0;
1979 }
1980
1981 /**
1982  * Validate META item.
1983  *
1984  * @param[in] dev
1985  *   Pointer to the rte_eth_dev structure.
1986  * @param[in] item
1987  *   Item specification.
1988  * @param[in] attr
1989  *   Attributes of flow that includes this item.
1990  * @param[out] error
1991  *   Pointer to error structure.
1992  *
1993  * @return
1994  *   0 on success, a negative errno value otherwise and rte_errno is set.
1995  */
1996 static int
1997 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1998                            const struct rte_flow_item *item,
1999                            const struct rte_flow_attr *attr,
2000                            struct rte_flow_error *error)
2001 {
2002         struct mlx5_priv *priv = dev->data->dev_private;
2003         struct mlx5_sh_config *config = &priv->sh->config;
2004         const struct rte_flow_item_meta *spec = item->spec;
2005         const struct rte_flow_item_meta *mask = item->mask;
2006         struct rte_flow_item_meta nic_mask = {
2007                 .data = UINT32_MAX
2008         };
2009         int reg;
2010         int ret;
2011
2012         if (!spec)
2013                 return rte_flow_error_set(error, EINVAL,
2014                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2015                                           item->spec,
2016                                           "data cannot be empty");
2017         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2018                 if (!mlx5_flow_ext_mreg_supported(dev))
2019                         return rte_flow_error_set(error, ENOTSUP,
2020                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2021                                           "extended metadata register"
2022                                           " isn't supported");
2023                 reg = flow_dv_get_metadata_reg(dev, attr, error);
2024                 if (reg < 0)
2025                         return reg;
2026                 if (reg == REG_NON)
2027                         return rte_flow_error_set(error, ENOTSUP,
2028                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2029                                         "unavailable extended metadata register");
2030                 if (reg == REG_B)
2031                         return rte_flow_error_set(error, ENOTSUP,
2032                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2033                                           "match on reg_b "
2034                                           "isn't supported");
2035                 if (reg != REG_A)
2036                         nic_mask.data = priv->sh->dv_meta_mask;
2037         } else {
2038                 if (attr->transfer)
2039                         return rte_flow_error_set(error, ENOTSUP,
2040                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2041                                         "extended metadata feature "
2042                                         "should be enabled when "
2043                                         "meta item is requested "
2044                                         "with e-switch mode ");
2045                 if (attr->ingress)
2046                         return rte_flow_error_set(error, ENOTSUP,
2047                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2048                                         "match on metadata for ingress "
2049                                         "is not supported in legacy "
2050                                         "metadata mode");
2051         }
2052         if (!mask)
2053                 mask = &rte_flow_item_meta_mask;
2054         if (!mask->data)
2055                 return rte_flow_error_set(error, EINVAL,
2056                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2057                                         "mask cannot be zero");
2058
2059         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2060                                         (const uint8_t *)&nic_mask,
2061                                         sizeof(struct rte_flow_item_meta),
2062                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2063         return ret;
2064 }
2065
2066 /**
2067  * Validate TAG item.
2068  *
2069  * @param[in] dev
2070  *   Pointer to the rte_eth_dev structure.
2071  * @param[in] item
2072  *   Item specification.
2073  * @param[in] attr
2074  *   Attributes of flow that includes this item.
2075  * @param[out] error
2076  *   Pointer to error structure.
2077  *
2078  * @return
2079  *   0 on success, a negative errno value otherwise and rte_errno is set.
2080  */
2081 static int
2082 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
2083                           const struct rte_flow_item *item,
2084                           const struct rte_flow_attr *attr __rte_unused,
2085                           struct rte_flow_error *error)
2086 {
2087         const struct rte_flow_item_tag *spec = item->spec;
2088         const struct rte_flow_item_tag *mask = item->mask;
2089         const struct rte_flow_item_tag nic_mask = {
2090                 .data = RTE_BE32(UINT32_MAX),
2091                 .index = 0xff,
2092         };
2093         int ret;
2094
2095         if (!mlx5_flow_ext_mreg_supported(dev))
2096                 return rte_flow_error_set(error, ENOTSUP,
2097                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2098                                           "extensive metadata register"
2099                                           " isn't supported");
2100         if (!spec)
2101                 return rte_flow_error_set(error, EINVAL,
2102                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2103                                           item->spec,
2104                                           "data cannot be empty");
2105         if (!mask)
2106                 mask = &rte_flow_item_tag_mask;
2107         if (!mask->data)
2108                 return rte_flow_error_set(error, EINVAL,
2109                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2110                                         "mask cannot be zero");
2111
2112         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2113                                         (const uint8_t *)&nic_mask,
2114                                         sizeof(struct rte_flow_item_tag),
2115                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2116         if (ret < 0)
2117                 return ret;
2118         if (mask->index != 0xff)
2119                 return rte_flow_error_set(error, EINVAL,
2120                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2121                                           "partial mask for tag index"
2122                                           " is not supported");
2123         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
2124         if (ret < 0)
2125                 return ret;
2126         MLX5_ASSERT(ret != REG_NON);
2127         return 0;
2128 }
2129
2130 /**
2131  * Validate vport item.
2132  *
2133  * @param[in] dev
2134  *   Pointer to the rte_eth_dev structure.
2135  * @param[in] item
2136  *   Item specification.
2137  * @param[in] attr
2138  *   Attributes of flow that includes this item.
2139  * @param[in] item_flags
2140  *   Bit-fields that holds the items detected until now.
2141  * @param[out] error
2142  *   Pointer to error structure.
2143  *
2144  * @return
2145  *   0 on success, a negative errno value otherwise and rte_errno is set.
2146  */
2147 static int
2148 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
2149                               const struct rte_flow_item *item,
2150                               const struct rte_flow_attr *attr,
2151                               uint64_t item_flags,
2152                               struct rte_flow_error *error)
2153 {
2154         const struct rte_flow_item_port_id *spec = item->spec;
2155         const struct rte_flow_item_port_id *mask = item->mask;
2156         const struct rte_flow_item_port_id switch_mask = {
2157                         .id = 0xffffffff,
2158         };
2159         struct mlx5_priv *esw_priv;
2160         struct mlx5_priv *dev_priv;
2161         int ret;
2162
2163         if (!attr->transfer)
2164                 return rte_flow_error_set(error, EINVAL,
2165                                           RTE_FLOW_ERROR_TYPE_ITEM,
2166                                           NULL,
2167                                           "match on port id is valid only"
2168                                           " when transfer flag is enabled");
2169         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
2170                 return rte_flow_error_set(error, ENOTSUP,
2171                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2172                                           "multiple source ports are not"
2173                                           " supported");
2174         if (!mask)
2175                 mask = &switch_mask;
2176         if (mask->id != 0xffffffff)
2177                 return rte_flow_error_set(error, ENOTSUP,
2178                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2179                                            mask,
2180                                            "no support for partial mask on"
2181                                            " \"id\" field");
2182         ret = mlx5_flow_item_acceptable
2183                                 (item, (const uint8_t *)mask,
2184                                  (const uint8_t *)&rte_flow_item_port_id_mask,
2185                                  sizeof(struct rte_flow_item_port_id),
2186                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2187         if (ret)
2188                 return ret;
2189         if (!spec)
2190                 return 0;
2191         if (spec->id == MLX5_PORT_ESW_MGR)
2192                 return 0;
2193         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
2194         if (!esw_priv)
2195                 return rte_flow_error_set(error, rte_errno,
2196                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2197                                           "failed to obtain E-Switch info for"
2198                                           " port");
2199         dev_priv = mlx5_dev_to_eswitch_info(dev);
2200         if (!dev_priv)
2201                 return rte_flow_error_set(error, rte_errno,
2202                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2203                                           NULL,
2204                                           "failed to obtain E-Switch info");
2205         if (esw_priv->domain_id != dev_priv->domain_id)
2206                 return rte_flow_error_set(error, EINVAL,
2207                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2208                                           "cannot match on a port from a"
2209                                           " different E-Switch");
2210         return 0;
2211 }
2212
2213 /**
2214  * Validate VLAN item.
2215  *
2216  * @param[in] item
2217  *   Item specification.
2218  * @param[in] item_flags
2219  *   Bit-fields that holds the items detected until now.
2220  * @param[in] dev
2221  *   Ethernet device flow is being created on.
2222  * @param[out] error
2223  *   Pointer to error structure.
2224  *
2225  * @return
2226  *   0 on success, a negative errno value otherwise and rte_errno is set.
2227  */
2228 static int
2229 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
2230                            uint64_t item_flags,
2231                            struct rte_eth_dev *dev,
2232                            struct rte_flow_error *error)
2233 {
2234         const struct rte_flow_item_vlan *mask = item->mask;
2235         const struct rte_flow_item_vlan nic_mask = {
2236                 .tci = RTE_BE16(UINT16_MAX),
2237                 .inner_type = RTE_BE16(UINT16_MAX),
2238                 .has_more_vlan = 1,
2239         };
2240         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2241         int ret;
2242         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2243                                         MLX5_FLOW_LAYER_INNER_L4) :
2244                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2245                                         MLX5_FLOW_LAYER_OUTER_L4);
2246         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2247                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2248
2249         if (item_flags & vlanm)
2250                 return rte_flow_error_set(error, EINVAL,
2251                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2252                                           "multiple VLAN layers not supported");
2253         else if ((item_flags & l34m) != 0)
2254                 return rte_flow_error_set(error, EINVAL,
2255                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2256                                           "VLAN cannot follow L3/L4 layer");
2257         if (!mask)
2258                 mask = &rte_flow_item_vlan_mask;
2259         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2260                                         (const uint8_t *)&nic_mask,
2261                                         sizeof(struct rte_flow_item_vlan),
2262                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2263         if (ret)
2264                 return ret;
2265         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2266                 struct mlx5_priv *priv = dev->data->dev_private;
2267
2268                 if (priv->vmwa_context) {
2269                         /*
2270                          * Non-NULL context means we have a virtual machine
2271                          * and SR-IOV enabled, we have to create VLAN interface
2272                          * to make hypervisor to setup E-Switch vport
2273                          * context correctly. We avoid creating the multiple
2274                          * VLAN interfaces, so we cannot support VLAN tag mask.
2275                          */
2276                         return rte_flow_error_set(error, EINVAL,
2277                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2278                                                   item,
2279                                                   "VLAN tag mask is not"
2280                                                   " supported in virtual"
2281                                                   " environment");
2282                 }
2283         }
2284         return 0;
2285 }
2286
2287 /*
2288  * GTP flags are contained in 1 byte of the format:
2289  * -------------------------------------------
2290  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
2291  * |-----------------------------------------|
2292  * | value | Version | PT | Res | E | S | PN |
2293  * -------------------------------------------
2294  *
2295  * Matching is supported only for GTP flags E, S, PN.
2296  */
2297 #define MLX5_GTP_FLAGS_MASK     0x07
2298
2299 /**
2300  * Validate GTP item.
2301  *
2302  * @param[in] dev
2303  *   Pointer to the rte_eth_dev structure.
2304  * @param[in] item
2305  *   Item specification.
2306  * @param[in] item_flags
2307  *   Bit-fields that holds the items detected until now.
2308  * @param[out] error
2309  *   Pointer to error structure.
2310  *
2311  * @return
2312  *   0 on success, a negative errno value otherwise and rte_errno is set.
2313  */
2314 static int
2315 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
2316                           const struct rte_flow_item *item,
2317                           uint64_t item_flags,
2318                           struct rte_flow_error *error)
2319 {
2320         struct mlx5_priv *priv = dev->data->dev_private;
2321         const struct rte_flow_item_gtp *spec = item->spec;
2322         const struct rte_flow_item_gtp *mask = item->mask;
2323         const struct rte_flow_item_gtp nic_mask = {
2324                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
2325                 .msg_type = 0xff,
2326                 .teid = RTE_BE32(0xffffffff),
2327         };
2328
2329         if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_gtp)
2330                 return rte_flow_error_set(error, ENOTSUP,
2331                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2332                                           "GTP support is not enabled");
2333         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2334                 return rte_flow_error_set(error, ENOTSUP,
2335                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2336                                           "multiple tunnel layers not"
2337                                           " supported");
2338         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2339                 return rte_flow_error_set(error, EINVAL,
2340                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2341                                           "no outer UDP layer found");
2342         if (!mask)
2343                 mask = &rte_flow_item_gtp_mask;
2344         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
2345                 return rte_flow_error_set(error, ENOTSUP,
2346                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2347                                           "Match is supported for GTP"
2348                                           " flags only");
2349         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2350                                          (const uint8_t *)&nic_mask,
2351                                          sizeof(struct rte_flow_item_gtp),
2352                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2353 }
2354
2355 /**
2356  * Validate GTP PSC item.
2357  *
2358  * @param[in] item
2359  *   Item specification.
2360  * @param[in] last_item
2361  *   Previous validated item in the pattern items.
2362  * @param[in] gtp_item
2363  *   Previous GTP item specification.
2364  * @param[in] attr
2365  *   Pointer to flow attributes.
2366  * @param[out] error
2367  *   Pointer to error structure.
2368  *
2369  * @return
2370  *   0 on success, a negative errno value otherwise and rte_errno is set.
2371  */
2372 static int
2373 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
2374                               uint64_t last_item,
2375                               const struct rte_flow_item *gtp_item,
2376                               const struct rte_flow_attr *attr,
2377                               struct rte_flow_error *error)
2378 {
2379         const struct rte_flow_item_gtp *gtp_spec;
2380         const struct rte_flow_item_gtp *gtp_mask;
2381         const struct rte_flow_item_gtp_psc *mask;
2382         const struct rte_flow_item_gtp_psc nic_mask = {
2383                 .hdr.type = 0xF,
2384                 .hdr.qfi = 0x3F,
2385         };
2386
2387         if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
2388                 return rte_flow_error_set
2389                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2390                          "GTP PSC item must be preceded with GTP item");
2391         gtp_spec = gtp_item->spec;
2392         gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
2393         /* GTP spec and E flag is requested to match zero. */
2394         if (gtp_spec &&
2395                 (gtp_mask->v_pt_rsv_flags &
2396                 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
2397                 return rte_flow_error_set
2398                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2399                          "GTP E flag must be 1 to match GTP PSC");
2400         /* Check the flow is not created in group zero. */
2401         if (!attr->transfer && !attr->group)
2402                 return rte_flow_error_set
2403                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2404                          "GTP PSC is not supported for group 0");
2405         /* GTP spec is here and E flag is requested to match zero. */
2406         if (!item->spec)
2407                 return 0;
2408         mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
2409         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2410                                          (const uint8_t *)&nic_mask,
2411                                          sizeof(struct rte_flow_item_gtp_psc),
2412                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2413 }
2414
2415 /**
2416  * Validate IPV4 item.
2417  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
2418  * add specific validation of fragment_offset field,
2419  *
2420  * @param[in] item
2421  *   Item specification.
2422  * @param[in] item_flags
2423  *   Bit-fields that holds the items detected until now.
2424  * @param[out] error
2425  *   Pointer to error structure.
2426  *
2427  * @return
2428  *   0 on success, a negative errno value otherwise and rte_errno is set.
2429  */
2430 static int
2431 flow_dv_validate_item_ipv4(struct rte_eth_dev *dev,
2432                            const struct rte_flow_item *item,
2433                            uint64_t item_flags, uint64_t last_item,
2434                            uint16_t ether_type, struct rte_flow_error *error)
2435 {
2436         int ret;
2437         struct mlx5_priv *priv = dev->data->dev_private;
2438         struct mlx5_hca_attr *attr = &priv->sh->cdev->config.hca_attr;
2439         const struct rte_flow_item_ipv4 *spec = item->spec;
2440         const struct rte_flow_item_ipv4 *last = item->last;
2441         const struct rte_flow_item_ipv4 *mask = item->mask;
2442         rte_be16_t fragment_offset_spec = 0;
2443         rte_be16_t fragment_offset_last = 0;
2444         struct rte_flow_item_ipv4 nic_ipv4_mask = {
2445                 .hdr = {
2446                         .src_addr = RTE_BE32(0xffffffff),
2447                         .dst_addr = RTE_BE32(0xffffffff),
2448                         .type_of_service = 0xff,
2449                         .fragment_offset = RTE_BE16(0xffff),
2450                         .next_proto_id = 0xff,
2451                         .time_to_live = 0xff,
2452                 },
2453         };
2454
2455         if (mask && (mask->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK)) {
2456                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2457                 bool ihl_cap = !tunnel ?
2458                                attr->outer_ipv4_ihl : attr->inner_ipv4_ihl;
2459                 if (!ihl_cap)
2460                         return rte_flow_error_set(error, ENOTSUP,
2461                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2462                                                   item,
2463                                                   "IPV4 ihl offload not supported");
2464                 nic_ipv4_mask.hdr.version_ihl = mask->hdr.version_ihl;
2465         }
2466         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
2467                                            ether_type, &nic_ipv4_mask,
2468                                            MLX5_ITEM_RANGE_ACCEPTED, error);
2469         if (ret < 0)
2470                 return ret;
2471         if (spec && mask)
2472                 fragment_offset_spec = spec->hdr.fragment_offset &
2473                                        mask->hdr.fragment_offset;
2474         if (!fragment_offset_spec)
2475                 return 0;
2476         /*
2477          * spec and mask are valid, enforce using full mask to make sure the
2478          * complete value is used correctly.
2479          */
2480         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2481                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2482                 return rte_flow_error_set(error, EINVAL,
2483                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2484                                           item, "must use full mask for"
2485                                           " fragment_offset");
2486         /*
2487          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
2488          * indicating this is 1st fragment of fragmented packet.
2489          * This is not yet supported in MLX5, return appropriate error message.
2490          */
2491         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
2492                 return rte_flow_error_set(error, ENOTSUP,
2493                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2494                                           "match on first fragment not "
2495                                           "supported");
2496         if (fragment_offset_spec && !last)
2497                 return rte_flow_error_set(error, ENOTSUP,
2498                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2499                                           "specified value not supported");
2500         /* spec and last are valid, validate the specified range. */
2501         fragment_offset_last = last->hdr.fragment_offset &
2502                                mask->hdr.fragment_offset;
2503         /*
2504          * Match on fragment_offset spec 0x2001 and last 0x3fff
2505          * means MF is 1 and frag-offset is > 0.
2506          * This packet is fragment 2nd and onward, excluding last.
2507          * This is not yet supported in MLX5, return appropriate
2508          * error message.
2509          */
2510         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
2511             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2512                 return rte_flow_error_set(error, ENOTSUP,
2513                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2514                                           last, "match on following "
2515                                           "fragments not supported");
2516         /*
2517          * Match on fragment_offset spec 0x0001 and last 0x1fff
2518          * means MF is 0 and frag-offset is > 0.
2519          * This packet is last fragment of fragmented packet.
2520          * This is not yet supported in MLX5, return appropriate
2521          * error message.
2522          */
2523         if (fragment_offset_spec == RTE_BE16(1) &&
2524             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
2525                 return rte_flow_error_set(error, ENOTSUP,
2526                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2527                                           last, "match on last "
2528                                           "fragment not supported");
2529         /*
2530          * Match on fragment_offset spec 0x0001 and last 0x3fff
2531          * means MF and/or frag-offset is not 0.
2532          * This is a fragmented packet.
2533          * Other range values are invalid and rejected.
2534          */
2535         if (!(fragment_offset_spec == RTE_BE16(1) &&
2536               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
2537                 return rte_flow_error_set(error, ENOTSUP,
2538                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2539                                           "specified range not supported");
2540         return 0;
2541 }
2542
2543 /**
2544  * Validate IPV6 fragment extension item.
2545  *
2546  * @param[in] item
2547  *   Item specification.
2548  * @param[in] item_flags
2549  *   Bit-fields that holds the items detected until now.
2550  * @param[out] error
2551  *   Pointer to error structure.
2552  *
2553  * @return
2554  *   0 on success, a negative errno value otherwise and rte_errno is set.
2555  */
2556 static int
2557 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
2558                                     uint64_t item_flags,
2559                                     struct rte_flow_error *error)
2560 {
2561         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
2562         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
2563         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
2564         rte_be16_t frag_data_spec = 0;
2565         rte_be16_t frag_data_last = 0;
2566         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2567         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2568                                       MLX5_FLOW_LAYER_OUTER_L4;
2569         int ret = 0;
2570         struct rte_flow_item_ipv6_frag_ext nic_mask = {
2571                 .hdr = {
2572                         .next_header = 0xff,
2573                         .frag_data = RTE_BE16(0xffff),
2574                 },
2575         };
2576
2577         if (item_flags & l4m)
2578                 return rte_flow_error_set(error, EINVAL,
2579                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2580                                           "ipv6 fragment extension item cannot "
2581                                           "follow L4 item.");
2582         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
2583             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2584                 return rte_flow_error_set(error, EINVAL,
2585                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2586                                           "ipv6 fragment extension item must "
2587                                           "follow ipv6 item");
2588         if (spec && mask)
2589                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
2590         if (!frag_data_spec)
2591                 return 0;
2592         /*
2593          * spec and mask are valid, enforce using full mask to make sure the
2594          * complete value is used correctly.
2595          */
2596         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
2597                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2598                 return rte_flow_error_set(error, EINVAL,
2599                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2600                                           item, "must use full mask for"
2601                                           " frag_data");
2602         /*
2603          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
2604          * This is 1st fragment of fragmented packet.
2605          */
2606         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
2607                 return rte_flow_error_set(error, ENOTSUP,
2608                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2609                                           "match on first fragment not "
2610                                           "supported");
2611         if (frag_data_spec && !last)
2612                 return rte_flow_error_set(error, EINVAL,
2613                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2614                                           "specified value not supported");
2615         ret = mlx5_flow_item_acceptable
2616                                 (item, (const uint8_t *)mask,
2617                                  (const uint8_t *)&nic_mask,
2618                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
2619                                  MLX5_ITEM_RANGE_ACCEPTED, error);
2620         if (ret)
2621                 return ret;
2622         /* spec and last are valid, validate the specified range. */
2623         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
2624         /*
2625          * Match on frag_data spec 0x0009 and last 0xfff9
2626          * means M is 1 and frag-offset is > 0.
2627          * This packet is fragment 2nd and onward, excluding last.
2628          * This is not yet supported in MLX5, return appropriate
2629          * error message.
2630          */
2631         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2632                                        RTE_IPV6_EHDR_MF_MASK) &&
2633             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2634                 return rte_flow_error_set(error, ENOTSUP,
2635                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2636                                           last, "match on following "
2637                                           "fragments not supported");
2638         /*
2639          * Match on frag_data spec 0x0008 and last 0xfff8
2640          * means M is 0 and frag-offset is > 0.
2641          * This packet is last fragment of fragmented packet.
2642          * This is not yet supported in MLX5, return appropriate
2643          * error message.
2644          */
2645         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2646             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2647                 return rte_flow_error_set(error, ENOTSUP,
2648                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2649                                           last, "match on last "
2650                                           "fragment not supported");
2651         /* Other range values are invalid and rejected. */
2652         return rte_flow_error_set(error, EINVAL,
2653                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2654                                   "specified range not supported");
2655 }
2656
2657 /*
2658  * Validate ASO CT item.
2659  *
2660  * @param[in] dev
2661  *   Pointer to the rte_eth_dev structure.
2662  * @param[in] item
2663  *   Item specification.
2664  * @param[in] item_flags
2665  *   Pointer to bit-fields that holds the items detected until now.
2666  * @param[out] error
2667  *   Pointer to error structure.
2668  *
2669  * @return
2670  *   0 on success, a negative errno value otherwise and rte_errno is set.
2671  */
2672 static int
2673 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
2674                              const struct rte_flow_item *item,
2675                              uint64_t *item_flags,
2676                              struct rte_flow_error *error)
2677 {
2678         const struct rte_flow_item_conntrack *spec = item->spec;
2679         const struct rte_flow_item_conntrack *mask = item->mask;
2680         RTE_SET_USED(dev);
2681         uint32_t flags;
2682
2683         if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
2684                 return rte_flow_error_set(error, EINVAL,
2685                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2686                                           "Only one CT is supported");
2687         if (!mask)
2688                 mask = &rte_flow_item_conntrack_mask;
2689         flags = spec->flags & mask->flags;
2690         if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
2691             ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
2692              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
2693              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
2694                 return rte_flow_error_set(error, EINVAL,
2695                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2696                                           "Conflict status bits");
2697         /* State change also needs to be considered. */
2698         *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
2699         return 0;
2700 }
2701
2702 /**
2703  * Validate the pop VLAN action.
2704  *
2705  * @param[in] dev
2706  *   Pointer to the rte_eth_dev structure.
2707  * @param[in] action_flags
2708  *   Holds the actions detected until now.
2709  * @param[in] action
2710  *   Pointer to the pop vlan action.
2711  * @param[in] item_flags
2712  *   The items found in this flow rule.
2713  * @param[in] attr
2714  *   Pointer to flow attributes.
2715  * @param[out] error
2716  *   Pointer to error structure.
2717  *
2718  * @return
2719  *   0 on success, a negative errno value otherwise and rte_errno is set.
2720  */
2721 static int
2722 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2723                                  uint64_t action_flags,
2724                                  const struct rte_flow_action *action,
2725                                  uint64_t item_flags,
2726                                  const struct rte_flow_attr *attr,
2727                                  struct rte_flow_error *error)
2728 {
2729         const struct mlx5_priv *priv = dev->data->dev_private;
2730         struct mlx5_dev_ctx_shared *sh = priv->sh;
2731         bool direction_error = false;
2732
2733         if (!priv->sh->pop_vlan_action)
2734                 return rte_flow_error_set(error, ENOTSUP,
2735                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2736                                           NULL,
2737                                           "pop vlan action is not supported");
2738         /* Pop VLAN is not supported in egress except for CX6 FDB mode. */
2739         if (attr->transfer) {
2740                 bool fdb_tx = priv->representor_id != UINT16_MAX;
2741                 bool is_cx5 = sh->steering_format_version ==
2742                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
2743
2744                 if (fdb_tx && is_cx5)
2745                         direction_error = true;
2746         } else if (attr->egress) {
2747                 direction_error = true;
2748         }
2749         if (direction_error)
2750                 return rte_flow_error_set(error, ENOTSUP,
2751                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2752                                           NULL,
2753                                           "pop vlan action not supported for egress");
2754         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2755                 return rte_flow_error_set(error, ENOTSUP,
2756                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2757                                           "no support for multiple VLAN "
2758                                           "actions");
2759         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2760         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2761             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2762                 return rte_flow_error_set(error, ENOTSUP,
2763                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2764                                           NULL,
2765                                           "cannot pop vlan after decap without "
2766                                           "match on inner vlan in the flow");
2767         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2768         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2769             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2770                 return rte_flow_error_set(error, ENOTSUP,
2771                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2772                                           NULL,
2773                                           "cannot pop vlan without a "
2774                                           "match on (outer) vlan in the flow");
2775         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2776                 return rte_flow_error_set(error, EINVAL,
2777                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2778                                           "wrong action order, port_id should "
2779                                           "be after pop VLAN action");
2780         if (!attr->transfer && priv->representor)
2781                 return rte_flow_error_set(error, ENOTSUP,
2782                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2783                                           "pop vlan action for VF representor "
2784                                           "not supported on NIC table");
2785         return 0;
2786 }
2787
2788 /**
2789  * Get VLAN default info from vlan match info.
2790  *
2791  * @param[in] items
2792  *   the list of item specifications.
2793  * @param[out] vlan
2794  *   pointer VLAN info to fill to.
2795  *
2796  * @return
2797  *   0 on success, a negative errno value otherwise and rte_errno is set.
2798  */
2799 static void
2800 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2801                                   struct rte_vlan_hdr *vlan)
2802 {
2803         const struct rte_flow_item_vlan nic_mask = {
2804                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2805                                 MLX5DV_FLOW_VLAN_VID_MASK),
2806                 .inner_type = RTE_BE16(0xffff),
2807         };
2808
2809         if (items == NULL)
2810                 return;
2811         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2812                 int type = items->type;
2813
2814                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2815                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2816                         break;
2817         }
2818         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2819                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2820                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2821
2822                 /* If VLAN item in pattern doesn't contain data, return here. */
2823                 if (!vlan_v)
2824                         return;
2825                 if (!vlan_m)
2826                         vlan_m = &nic_mask;
2827                 /* Only full match values are accepted */
2828                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2829                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2830                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2831                         vlan->vlan_tci |=
2832                                 rte_be_to_cpu_16(vlan_v->tci &
2833                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2834                 }
2835                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2836                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2837                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2838                         vlan->vlan_tci |=
2839                                 rte_be_to_cpu_16(vlan_v->tci &
2840                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2841                 }
2842                 if (vlan_m->inner_type == nic_mask.inner_type)
2843                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2844                                                            vlan_m->inner_type);
2845         }
2846 }
2847
2848 /**
2849  * Validate the push VLAN action.
2850  *
2851  * @param[in] dev
2852  *   Pointer to the rte_eth_dev structure.
2853  * @param[in] action_flags
2854  *   Holds the actions detected until now.
2855  * @param[in] item_flags
2856  *   The items found in this flow rule.
2857  * @param[in] action
2858  *   Pointer to the action structure.
2859  * @param[in] attr
2860  *   Pointer to flow attributes
2861  * @param[out] error
2862  *   Pointer to error structure.
2863  *
2864  * @return
2865  *   0 on success, a negative errno value otherwise and rte_errno is set.
2866  */
2867 static int
2868 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2869                                   uint64_t action_flags,
2870                                   const struct rte_flow_item_vlan *vlan_m,
2871                                   const struct rte_flow_action *action,
2872                                   const struct rte_flow_attr *attr,
2873                                   struct rte_flow_error *error)
2874 {
2875         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2876         const struct mlx5_priv *priv = dev->data->dev_private;
2877
2878         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2879             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2880                 return rte_flow_error_set(error, EINVAL,
2881                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2882                                           "invalid vlan ethertype");
2883         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2884                 return rte_flow_error_set(error, EINVAL,
2885                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2886                                           "wrong action order, port_id should "
2887                                           "be after push VLAN");
2888         if (!attr->transfer && priv->representor)
2889                 return rte_flow_error_set(error, ENOTSUP,
2890                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2891                                           "push vlan action for VF representor "
2892                                           "not supported on NIC table");
2893         if (vlan_m &&
2894             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2895             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2896                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2897             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2898             !(mlx5_flow_find_action
2899                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2900                 return rte_flow_error_set(error, EINVAL,
2901                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2902                                           "not full match mask on VLAN PCP and "
2903                                           "there is no of_set_vlan_pcp action, "
2904                                           "push VLAN action cannot figure out "
2905                                           "PCP value");
2906         if (vlan_m &&
2907             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2908             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2909                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2910             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2911             !(mlx5_flow_find_action
2912                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2913                 return rte_flow_error_set(error, EINVAL,
2914                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2915                                           "not full match mask on VLAN VID and "
2916                                           "there is no of_set_vlan_vid action, "
2917                                           "push VLAN action cannot figure out "
2918                                           "VID value");
2919         (void)attr;
2920         return 0;
2921 }
2922
2923 /**
2924  * Validate the set VLAN PCP.
2925  *
2926  * @param[in] action_flags
2927  *   Holds the actions detected until now.
2928  * @param[in] actions
2929  *   Pointer to the list of actions remaining in the flow rule.
2930  * @param[out] error
2931  *   Pointer to error structure.
2932  *
2933  * @return
2934  *   0 on success, a negative errno value otherwise and rte_errno is set.
2935  */
2936 static int
2937 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2938                                      const struct rte_flow_action actions[],
2939                                      struct rte_flow_error *error)
2940 {
2941         const struct rte_flow_action *action = actions;
2942         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2943
2944         if (conf->vlan_pcp > 7)
2945                 return rte_flow_error_set(error, EINVAL,
2946                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2947                                           "VLAN PCP value is too big");
2948         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2949                 return rte_flow_error_set(error, ENOTSUP,
2950                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2951                                           "set VLAN PCP action must follow "
2952                                           "the push VLAN action");
2953         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2954                 return rte_flow_error_set(error, ENOTSUP,
2955                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2956                                           "Multiple VLAN PCP modification are "
2957                                           "not supported");
2958         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2959                 return rte_flow_error_set(error, EINVAL,
2960                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2961                                           "wrong action order, port_id should "
2962                                           "be after set VLAN PCP");
2963         return 0;
2964 }
2965
2966 /**
2967  * Validate the set VLAN VID.
2968  *
2969  * @param[in] item_flags
2970  *   Holds the items detected in this rule.
2971  * @param[in] action_flags
2972  *   Holds the actions detected until now.
2973  * @param[in] actions
2974  *   Pointer to the list of actions remaining in the flow rule.
2975  * @param[out] error
2976  *   Pointer to error structure.
2977  *
2978  * @return
2979  *   0 on success, a negative errno value otherwise and rte_errno is set.
2980  */
2981 static int
2982 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2983                                      uint64_t action_flags,
2984                                      const struct rte_flow_action actions[],
2985                                      struct rte_flow_error *error)
2986 {
2987         const struct rte_flow_action *action = actions;
2988         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2989
2990         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2991                 return rte_flow_error_set(error, EINVAL,
2992                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2993                                           "VLAN VID value is too big");
2994         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2995             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2996                 return rte_flow_error_set(error, ENOTSUP,
2997                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2998                                           "set VLAN VID action must follow push"
2999                                           " VLAN action or match on VLAN item");
3000         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
3001                 return rte_flow_error_set(error, ENOTSUP,
3002                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3003                                           "Multiple VLAN VID modifications are "
3004                                           "not supported");
3005         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3006                 return rte_flow_error_set(error, EINVAL,
3007                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3008                                           "wrong action order, port_id should "
3009                                           "be after set VLAN VID");
3010         return 0;
3011 }
3012
3013 /*
3014  * Validate the FLAG action.
3015  *
3016  * @param[in] dev
3017  *   Pointer to the rte_eth_dev structure.
3018  * @param[in] action_flags
3019  *   Holds the actions detected until now.
3020  * @param[in] attr
3021  *   Pointer to flow attributes
3022  * @param[out] error
3023  *   Pointer to error structure.
3024  *
3025  * @return
3026  *   0 on success, a negative errno value otherwise and rte_errno is set.
3027  */
3028 static int
3029 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
3030                              uint64_t action_flags,
3031                              const struct rte_flow_attr *attr,
3032                              struct rte_flow_error *error)
3033 {
3034         struct mlx5_priv *priv = dev->data->dev_private;
3035         struct mlx5_sh_config *config = &priv->sh->config;
3036         int ret;
3037
3038         /* Fall back if no extended metadata register support. */
3039         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3040                 return mlx5_flow_validate_action_flag(action_flags, attr,
3041                                                       error);
3042         /* Extensive metadata mode requires registers. */
3043         if (!mlx5_flow_ext_mreg_supported(dev))
3044                 return rte_flow_error_set(error, ENOTSUP,
3045                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3046                                           "no metadata registers "
3047                                           "to support flag action");
3048         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
3049                 return rte_flow_error_set(error, ENOTSUP,
3050                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3051                                           "extended metadata register"
3052                                           " isn't available");
3053         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3054         if (ret < 0)
3055                 return ret;
3056         MLX5_ASSERT(ret > 0);
3057         if (action_flags & MLX5_FLOW_ACTION_MARK)
3058                 return rte_flow_error_set(error, EINVAL,
3059                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3060                                           "can't mark and flag in same flow");
3061         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3062                 return rte_flow_error_set(error, EINVAL,
3063                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3064                                           "can't have 2 flag"
3065                                           " actions in same flow");
3066         return 0;
3067 }
3068
3069 /**
3070  * Validate MARK action.
3071  *
3072  * @param[in] dev
3073  *   Pointer to the rte_eth_dev structure.
3074  * @param[in] action
3075  *   Pointer to action.
3076  * @param[in] action_flags
3077  *   Holds the actions detected until now.
3078  * @param[in] attr
3079  *   Pointer to flow attributes
3080  * @param[out] error
3081  *   Pointer to error structure.
3082  *
3083  * @return
3084  *   0 on success, a negative errno value otherwise and rte_errno is set.
3085  */
3086 static int
3087 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
3088                              const struct rte_flow_action *action,
3089                              uint64_t action_flags,
3090                              const struct rte_flow_attr *attr,
3091                              struct rte_flow_error *error)
3092 {
3093         struct mlx5_priv *priv = dev->data->dev_private;
3094         struct mlx5_sh_config *config = &priv->sh->config;
3095         const struct rte_flow_action_mark *mark = action->conf;
3096         int ret;
3097
3098         if (is_tunnel_offload_active(dev))
3099                 return rte_flow_error_set(error, ENOTSUP,
3100                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3101                                           "no mark action "
3102                                           "if tunnel offload active");
3103         /* Fall back if no extended metadata register support. */
3104         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3105                 return mlx5_flow_validate_action_mark(action, action_flags,
3106                                                       attr, error);
3107         /* Extensive metadata mode requires registers. */
3108         if (!mlx5_flow_ext_mreg_supported(dev))
3109                 return rte_flow_error_set(error, ENOTSUP,
3110                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3111                                           "no metadata registers "
3112                                           "to support mark action");
3113         if (!priv->sh->dv_mark_mask)
3114                 return rte_flow_error_set(error, ENOTSUP,
3115                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3116                                           "extended metadata register"
3117                                           " isn't available");
3118         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3119         if (ret < 0)
3120                 return ret;
3121         MLX5_ASSERT(ret > 0);
3122         if (!mark)
3123                 return rte_flow_error_set(error, EINVAL,
3124                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3125                                           "configuration cannot be null");
3126         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
3127                 return rte_flow_error_set(error, EINVAL,
3128                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3129                                           &mark->id,
3130                                           "mark id exceeds the limit");
3131         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3132                 return rte_flow_error_set(error, EINVAL,
3133                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3134                                           "can't flag and mark in same flow");
3135         if (action_flags & MLX5_FLOW_ACTION_MARK)
3136                 return rte_flow_error_set(error, EINVAL,
3137                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3138                                           "can't have 2 mark actions in same"
3139                                           " flow");
3140         return 0;
3141 }
3142
3143 /**
3144  * Validate SET_META action.
3145  *
3146  * @param[in] dev
3147  *   Pointer to the rte_eth_dev structure.
3148  * @param[in] action
3149  *   Pointer to the action structure.
3150  * @param[in] action_flags
3151  *   Holds the actions detected until now.
3152  * @param[in] attr
3153  *   Pointer to flow attributes
3154  * @param[out] error
3155  *   Pointer to error structure.
3156  *
3157  * @return
3158  *   0 on success, a negative errno value otherwise and rte_errno is set.
3159  */
3160 static int
3161 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
3162                                  const struct rte_flow_action *action,
3163                                  uint64_t action_flags __rte_unused,
3164                                  const struct rte_flow_attr *attr,
3165                                  struct rte_flow_error *error)
3166 {
3167         struct mlx5_priv *priv = dev->data->dev_private;
3168         struct mlx5_sh_config *config = &priv->sh->config;
3169         const struct rte_flow_action_set_meta *conf;
3170         uint32_t nic_mask = UINT32_MAX;
3171         int reg;
3172
3173         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
3174             !mlx5_flow_ext_mreg_supported(dev))
3175                 return rte_flow_error_set(error, ENOTSUP,
3176                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3177                                           "extended metadata register"
3178                                           " isn't supported");
3179         reg = flow_dv_get_metadata_reg(dev, attr, error);
3180         if (reg < 0)
3181                 return reg;
3182         if (reg == REG_NON)
3183                 return rte_flow_error_set(error, ENOTSUP,
3184                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3185                                           "unavailable extended metadata register");
3186         if (reg != REG_A && reg != REG_B) {
3187                 struct mlx5_priv *priv = dev->data->dev_private;
3188
3189                 nic_mask = priv->sh->dv_meta_mask;
3190         }
3191         if (!(action->conf))
3192                 return rte_flow_error_set(error, EINVAL,
3193                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3194                                           "configuration cannot be null");
3195         conf = (const struct rte_flow_action_set_meta *)action->conf;
3196         if (!conf->mask)
3197                 return rte_flow_error_set(error, EINVAL,
3198                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3199                                           "zero mask doesn't have any effect");
3200         if (conf->mask & ~nic_mask)
3201                 return rte_flow_error_set(error, EINVAL,
3202                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3203                                           "meta data must be within reg C0");
3204         return 0;
3205 }
3206
3207 /**
3208  * Validate SET_TAG action.
3209  *
3210  * @param[in] dev
3211  *   Pointer to the rte_eth_dev structure.
3212  * @param[in] action
3213  *   Pointer to the action structure.
3214  * @param[in] action_flags
3215  *   Holds the actions detected until now.
3216  * @param[in] attr
3217  *   Pointer to flow attributes
3218  * @param[out] error
3219  *   Pointer to error structure.
3220  *
3221  * @return
3222  *   0 on success, a negative errno value otherwise and rte_errno is set.
3223  */
3224 static int
3225 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
3226                                 const struct rte_flow_action *action,
3227                                 uint64_t action_flags,
3228                                 const struct rte_flow_attr *attr,
3229                                 struct rte_flow_error *error)
3230 {
3231         const struct rte_flow_action_set_tag *conf;
3232         const uint64_t terminal_action_flags =
3233                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
3234                 MLX5_FLOW_ACTION_RSS;
3235         int ret;
3236
3237         if (!mlx5_flow_ext_mreg_supported(dev))
3238                 return rte_flow_error_set(error, ENOTSUP,
3239                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3240                                           "extensive metadata register"
3241                                           " isn't supported");
3242         if (!(action->conf))
3243                 return rte_flow_error_set(error, EINVAL,
3244                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3245                                           "configuration cannot be null");
3246         conf = (const struct rte_flow_action_set_tag *)action->conf;
3247         if (!conf->mask)
3248                 return rte_flow_error_set(error, EINVAL,
3249                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3250                                           "zero mask doesn't have any effect");
3251         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
3252         if (ret < 0)
3253                 return ret;
3254         if (!attr->transfer && attr->ingress &&
3255             (action_flags & terminal_action_flags))
3256                 return rte_flow_error_set(error, EINVAL,
3257                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3258                                           "set_tag has no effect"
3259                                           " with terminal actions");
3260         return 0;
3261 }
3262
3263 /**
3264  * Indicates whether ASO aging is supported.
3265  *
3266  * @param[in] sh
3267  *   Pointer to shared device context structure.
3268  * @param[in] attr
3269  *   Attributes of flow that includes AGE action.
3270  *
3271  * @return
3272  *   True when ASO aging is supported, false otherwise.
3273  */
3274 static inline bool
3275 flow_hit_aso_supported(const struct mlx5_dev_ctx_shared *sh,
3276                 const struct rte_flow_attr *attr)
3277 {
3278         MLX5_ASSERT(sh && attr);
3279         return (sh->flow_hit_aso_en && (attr->transfer || attr->group));
3280 }
3281
3282 /**
3283  * Validate count action.
3284  *
3285  * @param[in] dev
3286  *   Pointer to rte_eth_dev structure.
3287  * @param[in] shared
3288  *   Indicator if action is shared.
3289  * @param[in] action_flags
3290  *   Holds the actions detected until now.
3291  * @param[in] attr
3292  *   Attributes of flow that includes this action.
3293  * @param[out] error
3294  *   Pointer to error structure.
3295  *
3296  * @return
3297  *   0 on success, a negative errno value otherwise and rte_errno is set.
3298  */
3299 static int
3300 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
3301                               uint64_t action_flags,
3302                               const struct rte_flow_attr *attr,
3303                               struct rte_flow_error *error)
3304 {
3305         struct mlx5_priv *priv = dev->data->dev_private;
3306
3307         if (!priv->sh->cdev->config.devx)
3308                 goto notsup_err;
3309         if (action_flags & MLX5_FLOW_ACTION_COUNT)
3310                 return rte_flow_error_set(error, EINVAL,
3311                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3312                                           "duplicate count actions set");
3313         if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
3314             !flow_hit_aso_supported(priv->sh, attr))
3315                 return rte_flow_error_set(error, EINVAL,
3316                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3317                                           "old age and indirect count combination is not supported");
3318 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
3319         return 0;
3320 #endif
3321 notsup_err:
3322         return rte_flow_error_set
3323                       (error, ENOTSUP,
3324                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3325                        NULL,
3326                        "count action not supported");
3327 }
3328
3329 /**
3330  * Validate the L2 encap action.
3331  *
3332  * @param[in] dev
3333  *   Pointer to the rte_eth_dev structure.
3334  * @param[in] action_flags
3335  *   Holds the actions detected until now.
3336  * @param[in] action
3337  *   Pointer to the action structure.
3338  * @param[in] attr
3339  *   Pointer to flow attributes.
3340  * @param[out] error
3341  *   Pointer to error structure.
3342  *
3343  * @return
3344  *   0 on success, a negative errno value otherwise and rte_errno is set.
3345  */
3346 static int
3347 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
3348                                  uint64_t action_flags,
3349                                  const struct rte_flow_action *action,
3350                                  const struct rte_flow_attr *attr,
3351                                  struct rte_flow_error *error)
3352 {
3353         const struct mlx5_priv *priv = dev->data->dev_private;
3354
3355         if (!(action->conf))
3356                 return rte_flow_error_set(error, EINVAL,
3357                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3358                                           "configuration cannot be null");
3359         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3360                 return rte_flow_error_set(error, EINVAL,
3361                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3362                                           "can only have a single encap action "
3363                                           "in a flow");
3364         if (!attr->transfer && priv->representor)
3365                 return rte_flow_error_set(error, ENOTSUP,
3366                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3367                                           "encap action for VF representor "
3368                                           "not supported on NIC table");
3369         return 0;
3370 }
3371
3372 /**
3373  * Validate a decap action.
3374  *
3375  * @param[in] dev
3376  *   Pointer to the rte_eth_dev structure.
3377  * @param[in] action_flags
3378  *   Holds the actions detected until now.
3379  * @param[in] action
3380  *   Pointer to the action structure.
3381  * @param[in] item_flags
3382  *   Holds the items detected.
3383  * @param[in] attr
3384  *   Pointer to flow attributes
3385  * @param[out] error
3386  *   Pointer to error structure.
3387  *
3388  * @return
3389  *   0 on success, a negative errno value otherwise and rte_errno is set.
3390  */
3391 static int
3392 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
3393                               uint64_t action_flags,
3394                               const struct rte_flow_action *action,
3395                               const uint64_t item_flags,
3396                               const struct rte_flow_attr *attr,
3397                               struct rte_flow_error *error)
3398 {
3399         const struct mlx5_priv *priv = dev->data->dev_private;
3400
3401         if (priv->sh->cdev->config.hca_attr.scatter_fcs_w_decap_disable &&
3402             !priv->sh->config.decap_en)
3403                 return rte_flow_error_set(error, ENOTSUP,
3404                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3405                                           "decap is not enabled");
3406         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
3407                 return rte_flow_error_set(error, ENOTSUP,
3408                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3409                                           action_flags &
3410                                           MLX5_FLOW_ACTION_DECAP ? "can only "
3411                                           "have a single decap action" : "decap "
3412                                           "after encap is not supported");
3413         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
3414                 return rte_flow_error_set(error, EINVAL,
3415                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3416                                           "can't have decap action after"
3417                                           " modify action");
3418         if (attr->egress)
3419                 return rte_flow_error_set(error, ENOTSUP,
3420                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
3421                                           NULL,
3422                                           "decap action not supported for "
3423                                           "egress");
3424         if (!attr->transfer && priv->representor)
3425                 return rte_flow_error_set(error, ENOTSUP,
3426                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3427                                           "decap action for VF representor "
3428                                           "not supported on NIC table");
3429         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
3430             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
3431                 return rte_flow_error_set(error, ENOTSUP,
3432                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3433                                 "VXLAN item should be present for VXLAN decap");
3434         return 0;
3435 }
3436
3437 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
3438
3439 /**
3440  * Validate the raw encap and decap actions.
3441  *
3442  * @param[in] dev
3443  *   Pointer to the rte_eth_dev structure.
3444  * @param[in] decap
3445  *   Pointer to the decap action.
3446  * @param[in] encap
3447  *   Pointer to the encap action.
3448  * @param[in] attr
3449  *   Pointer to flow attributes
3450  * @param[in/out] action_flags
3451  *   Holds the actions detected until now.
3452  * @param[out] actions_n
3453  *   pointer to the number of actions counter.
3454  * @param[in] action
3455  *   Pointer to the action structure.
3456  * @param[in] item_flags
3457  *   Holds the items detected.
3458  * @param[out] error
3459  *   Pointer to error structure.
3460  *
3461  * @return
3462  *   0 on success, a negative errno value otherwise and rte_errno is set.
3463  */
3464 static int
3465 flow_dv_validate_action_raw_encap_decap
3466         (struct rte_eth_dev *dev,
3467          const struct rte_flow_action_raw_decap *decap,
3468          const struct rte_flow_action_raw_encap *encap,
3469          const struct rte_flow_attr *attr, uint64_t *action_flags,
3470          int *actions_n, const struct rte_flow_action *action,
3471          uint64_t item_flags, struct rte_flow_error *error)
3472 {
3473         const struct mlx5_priv *priv = dev->data->dev_private;
3474         int ret;
3475
3476         if (encap && (!encap->size || !encap->data))
3477                 return rte_flow_error_set(error, EINVAL,
3478                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3479                                           "raw encap data cannot be empty");
3480         if (decap && encap) {
3481                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
3482                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3483                         /* L3 encap. */
3484                         decap = NULL;
3485                 else if (encap->size <=
3486                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3487                            decap->size >
3488                            MLX5_ENCAPSULATION_DECISION_SIZE)
3489                         /* L3 decap. */
3490                         encap = NULL;
3491                 else if (encap->size >
3492                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3493                            decap->size >
3494                            MLX5_ENCAPSULATION_DECISION_SIZE)
3495                         /* 2 L2 actions: encap and decap. */
3496                         ;
3497                 else
3498                         return rte_flow_error_set(error,
3499                                 ENOTSUP,
3500                                 RTE_FLOW_ERROR_TYPE_ACTION,
3501                                 NULL, "unsupported too small "
3502                                 "raw decap and too small raw "
3503                                 "encap combination");
3504         }
3505         if (decap) {
3506                 ret = flow_dv_validate_action_decap(dev, *action_flags, action,
3507                                                     item_flags, attr, error);
3508                 if (ret < 0)
3509                         return ret;
3510                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
3511                 ++(*actions_n);
3512         }
3513         if (encap) {
3514                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
3515                         return rte_flow_error_set(error, ENOTSUP,
3516                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3517                                                   NULL,
3518                                                   "small raw encap size");
3519                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
3520                         return rte_flow_error_set(error, EINVAL,
3521                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3522                                                   NULL,
3523                                                   "more than one encap action");
3524                 if (!attr->transfer && priv->representor)
3525                         return rte_flow_error_set
3526                                         (error, ENOTSUP,
3527                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3528                                          "encap action for VF representor "
3529                                          "not supported on NIC table");
3530                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
3531                 ++(*actions_n);
3532         }
3533         return 0;
3534 }
3535
3536 /*
3537  * Validate the ASO CT action.
3538  *
3539  * @param[in] dev
3540  *   Pointer to the rte_eth_dev structure.
3541  * @param[in] action_flags
3542  *   Holds the actions detected until now.
3543  * @param[in] item_flags
3544  *   The items found in this flow rule.
3545  * @param[in] attr
3546  *   Pointer to flow attributes.
3547  * @param[out] error
3548  *   Pointer to error structure.
3549  *
3550  * @return
3551  *   0 on success, a negative errno value otherwise and rte_errno is set.
3552  */
3553 static int
3554 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
3555                                uint64_t action_flags,
3556                                uint64_t item_flags,
3557                                const struct rte_flow_attr *attr,
3558                                struct rte_flow_error *error)
3559 {
3560         RTE_SET_USED(dev);
3561
3562         if (attr->group == 0 && !attr->transfer)
3563                 return rte_flow_error_set(error, ENOTSUP,
3564                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3565                                           NULL,
3566                                           "Only support non-root table");
3567         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
3568                 return rte_flow_error_set(error, ENOTSUP,
3569                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3570                                           "CT cannot follow a fate action");
3571         if ((action_flags & MLX5_FLOW_ACTION_METER) ||
3572             (action_flags & MLX5_FLOW_ACTION_AGE))
3573                 return rte_flow_error_set(error, EINVAL,
3574                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3575                                           "Only one ASO action is supported");
3576         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3577                 return rte_flow_error_set(error, EINVAL,
3578                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3579                                           "Encap cannot exist before CT");
3580         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3581                 return rte_flow_error_set(error, EINVAL,
3582                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3583                                           "Not a outer TCP packet");
3584         return 0;
3585 }
3586
3587 int
3588 flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
3589                              struct mlx5_list_entry *entry, void *cb_ctx)
3590 {
3591         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3592         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3593         struct mlx5_flow_dv_encap_decap_resource *resource;
3594
3595         resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
3596                                 entry);
3597         if (resource->reformat_type == ctx_resource->reformat_type &&
3598             resource->ft_type == ctx_resource->ft_type &&
3599             resource->flags == ctx_resource->flags &&
3600             resource->size == ctx_resource->size &&
3601             !memcmp((const void *)resource->buf,
3602                     (const void *)ctx_resource->buf,
3603                     resource->size))
3604                 return 0;
3605         return -1;
3606 }
3607
3608 struct mlx5_list_entry *
3609 flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
3610 {
3611         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3612         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3613         struct mlx5dv_dr_domain *domain;
3614         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3615         struct mlx5_flow_dv_encap_decap_resource *resource;
3616         uint32_t idx;
3617         int ret;
3618
3619         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3620                 domain = sh->fdb_domain;
3621         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3622                 domain = sh->rx_domain;
3623         else
3624                 domain = sh->tx_domain;
3625         /* Register new encap/decap resource. */
3626         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
3627         if (!resource) {
3628                 rte_flow_error_set(ctx->error, ENOMEM,
3629                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3630                                    "cannot allocate resource memory");
3631                 return NULL;
3632         }
3633         *resource = *ctx_resource;
3634         resource->idx = idx;
3635         ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->cdev->ctx,
3636                                                               domain, resource,
3637                                                              &resource->action);
3638         if (ret) {
3639                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
3640                 rte_flow_error_set(ctx->error, ENOMEM,
3641                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3642                                    NULL, "cannot create action");
3643                 return NULL;
3644         }
3645
3646         return &resource->entry;
3647 }
3648
3649 struct mlx5_list_entry *
3650 flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
3651                              void *cb_ctx)
3652 {
3653         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3654         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3655         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3656         uint32_t idx;
3657
3658         cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
3659                                            &idx);
3660         if (!cache_resource) {
3661                 rte_flow_error_set(ctx->error, ENOMEM,
3662                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3663                                    "cannot allocate resource memory");
3664                 return NULL;
3665         }
3666         memcpy(cache_resource, oentry, sizeof(*cache_resource));
3667         cache_resource->idx = idx;
3668         return &cache_resource->entry;
3669 }
3670
3671 void
3672 flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3673 {
3674         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3675         struct mlx5_flow_dv_encap_decap_resource *res =
3676                                        container_of(entry, typeof(*res), entry);
3677
3678         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
3679 }
3680
3681 /**
3682  * Find existing encap/decap resource or create and register a new one.
3683  *
3684  * @param[in, out] dev
3685  *   Pointer to rte_eth_dev structure.
3686  * @param[in, out] resource
3687  *   Pointer to encap/decap resource.
3688  * @parm[in, out] dev_flow
3689  *   Pointer to the dev_flow.
3690  * @param[out] error
3691  *   pointer to error structure.
3692  *
3693  * @return
3694  *   0 on success otherwise -errno and errno is set.
3695  */
3696 static int
3697 flow_dv_encap_decap_resource_register
3698                         (struct rte_eth_dev *dev,
3699                          struct mlx5_flow_dv_encap_decap_resource *resource,
3700                          struct mlx5_flow *dev_flow,
3701                          struct rte_flow_error *error)
3702 {
3703         struct mlx5_priv *priv = dev->data->dev_private;
3704         struct mlx5_dev_ctx_shared *sh = priv->sh;
3705         struct mlx5_list_entry *entry;
3706         union {
3707                 struct {
3708                         uint32_t ft_type:8;
3709                         uint32_t refmt_type:8;
3710                         /*
3711                          * Header reformat actions can be shared between
3712                          * non-root tables. One bit to indicate non-root
3713                          * table or not.
3714                          */
3715                         uint32_t is_root:1;
3716                         uint32_t reserve:15;
3717                 };
3718                 uint32_t v32;
3719         } encap_decap_key = {
3720                 {
3721                         .ft_type = resource->ft_type,
3722                         .refmt_type = resource->reformat_type,
3723                         .is_root = !!dev_flow->dv.group,
3724                         .reserve = 0,
3725                 }
3726         };
3727         struct mlx5_flow_cb_ctx ctx = {
3728                 .error = error,
3729                 .data = resource,
3730         };
3731         struct mlx5_hlist *encaps_decaps;
3732         uint64_t key64;
3733
3734         encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
3735                                 "encaps_decaps",
3736                                 MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
3737                                 true, true, sh,
3738                                 flow_dv_encap_decap_create_cb,
3739                                 flow_dv_encap_decap_match_cb,
3740                                 flow_dv_encap_decap_remove_cb,
3741                                 flow_dv_encap_decap_clone_cb,
3742                                 flow_dv_encap_decap_clone_free_cb,
3743                                 error);
3744         if (unlikely(!encaps_decaps))
3745                 return -rte_errno;
3746         resource->flags = dev_flow->dv.group ? 0 : 1;
3747         key64 =  __rte_raw_cksum(&encap_decap_key.v32,
3748                                  sizeof(encap_decap_key.v32), 0);
3749         if (resource->reformat_type !=
3750             MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
3751             resource->size)
3752                 key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
3753         entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
3754         if (!entry)
3755                 return -rte_errno;
3756         resource = container_of(entry, typeof(*resource), entry);
3757         dev_flow->dv.encap_decap = resource;
3758         dev_flow->handle->dvh.rix_encap_decap = resource->idx;
3759         return 0;
3760 }
3761
3762 /**
3763  * Find existing table jump resource or create and register a new one.
3764  *
3765  * @param[in, out] dev
3766  *   Pointer to rte_eth_dev structure.
3767  * @param[in, out] tbl
3768  *   Pointer to flow table resource.
3769  * @parm[in, out] dev_flow
3770  *   Pointer to the dev_flow.
3771  * @param[out] error
3772  *   pointer to error structure.
3773  *
3774  * @return
3775  *   0 on success otherwise -errno and errno is set.
3776  */
3777 static int
3778 flow_dv_jump_tbl_resource_register
3779                         (struct rte_eth_dev *dev __rte_unused,
3780                          struct mlx5_flow_tbl_resource *tbl,
3781                          struct mlx5_flow *dev_flow,
3782                          struct rte_flow_error *error __rte_unused)
3783 {
3784         struct mlx5_flow_tbl_data_entry *tbl_data =
3785                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
3786
3787         MLX5_ASSERT(tbl);
3788         MLX5_ASSERT(tbl_data->jump.action);
3789         dev_flow->handle->rix_jump = tbl_data->idx;
3790         dev_flow->dv.jump = &tbl_data->jump;
3791         return 0;
3792 }
3793
3794 int
3795 flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
3796                          struct mlx5_list_entry *entry, void *cb_ctx)
3797 {
3798         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3799         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3800         struct mlx5_flow_dv_port_id_action_resource *res =
3801                                        container_of(entry, typeof(*res), entry);
3802
3803         return ref->port_id != res->port_id;
3804 }
3805
3806 struct mlx5_list_entry *
3807 flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
3808 {
3809         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3810         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3811         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3812         struct mlx5_flow_dv_port_id_action_resource *resource;
3813         uint32_t idx;
3814         int ret;
3815
3816         /* Register new port id action resource. */
3817         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3818         if (!resource) {
3819                 rte_flow_error_set(ctx->error, ENOMEM,
3820                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3821                                    "cannot allocate port_id action memory");
3822                 return NULL;
3823         }
3824         *resource = *ref;
3825         ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
3826                                                         ref->port_id,
3827                                                         &resource->action);
3828         if (ret) {
3829                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
3830                 rte_flow_error_set(ctx->error, ENOMEM,
3831                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3832                                    "cannot create action");
3833                 return NULL;
3834         }
3835         resource->idx = idx;
3836         return &resource->entry;
3837 }
3838
3839 struct mlx5_list_entry *
3840 flow_dv_port_id_clone_cb(void *tool_ctx,
3841                          struct mlx5_list_entry *entry __rte_unused,
3842                          void *cb_ctx)
3843 {
3844         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3845         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3846         struct mlx5_flow_dv_port_id_action_resource *resource;
3847         uint32_t idx;
3848
3849         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3850         if (!resource) {
3851                 rte_flow_error_set(ctx->error, ENOMEM,
3852                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3853                                    "cannot allocate port_id action memory");
3854                 return NULL;
3855         }
3856         memcpy(resource, entry, sizeof(*resource));
3857         resource->idx = idx;
3858         return &resource->entry;
3859 }
3860
3861 void
3862 flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3863 {
3864         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3865         struct mlx5_flow_dv_port_id_action_resource *resource =
3866                                   container_of(entry, typeof(*resource), entry);
3867
3868         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
3869 }
3870
3871 /**
3872  * Find existing table port ID resource or create and register a new one.
3873  *
3874  * @param[in, out] dev
3875  *   Pointer to rte_eth_dev structure.
3876  * @param[in, out] ref
3877  *   Pointer to port ID action resource reference.
3878  * @parm[in, out] dev_flow
3879  *   Pointer to the dev_flow.
3880  * @param[out] error
3881  *   pointer to error structure.
3882  *
3883  * @return
3884  *   0 on success otherwise -errno and errno is set.
3885  */
3886 static int
3887 flow_dv_port_id_action_resource_register
3888                         (struct rte_eth_dev *dev,
3889                          struct mlx5_flow_dv_port_id_action_resource *ref,
3890                          struct mlx5_flow *dev_flow,
3891                          struct rte_flow_error *error)
3892 {
3893         struct mlx5_priv *priv = dev->data->dev_private;
3894         struct mlx5_list_entry *entry;
3895         struct mlx5_flow_dv_port_id_action_resource *resource;
3896         struct mlx5_flow_cb_ctx ctx = {
3897                 .error = error,
3898                 .data = ref,
3899         };
3900
3901         entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
3902         if (!entry)
3903                 return -rte_errno;
3904         resource = container_of(entry, typeof(*resource), entry);
3905         dev_flow->dv.port_id_action = resource;
3906         dev_flow->handle->rix_port_id_action = resource->idx;
3907         return 0;
3908 }
3909
3910 int
3911 flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
3912                            struct mlx5_list_entry *entry, void *cb_ctx)
3913 {
3914         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3915         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3916         struct mlx5_flow_dv_push_vlan_action_resource *res =
3917                                        container_of(entry, typeof(*res), entry);
3918
3919         return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
3920 }
3921
3922 struct mlx5_list_entry *
3923 flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
3924 {
3925         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3926         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3927         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3928         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3929         struct mlx5dv_dr_domain *domain;
3930         uint32_t idx;
3931         int ret;
3932
3933         /* Register new port id action resource. */
3934         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3935         if (!resource) {
3936                 rte_flow_error_set(ctx->error, ENOMEM,
3937                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3938                                    "cannot allocate push_vlan action memory");
3939                 return NULL;
3940         }
3941         *resource = *ref;
3942         if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3943                 domain = sh->fdb_domain;
3944         else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3945                 domain = sh->rx_domain;
3946         else
3947                 domain = sh->tx_domain;
3948         ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
3949                                                         &resource->action);
3950         if (ret) {
3951                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
3952                 rte_flow_error_set(ctx->error, ENOMEM,
3953                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3954                                    "cannot create push vlan action");
3955                 return NULL;
3956         }
3957         resource->idx = idx;
3958         return &resource->entry;
3959 }
3960
3961 struct mlx5_list_entry *
3962 flow_dv_push_vlan_clone_cb(void *tool_ctx,
3963                            struct mlx5_list_entry *entry __rte_unused,
3964                            void *cb_ctx)
3965 {
3966         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3967         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3968         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3969         uint32_t idx;
3970
3971         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3972         if (!resource) {
3973                 rte_flow_error_set(ctx->error, ENOMEM,
3974                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3975                                    "cannot allocate push_vlan action memory");
3976                 return NULL;
3977         }
3978         memcpy(resource, entry, sizeof(*resource));
3979         resource->idx = idx;
3980         return &resource->entry;
3981 }
3982
3983 void
3984 flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3985 {
3986         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3987         struct mlx5_flow_dv_push_vlan_action_resource *resource =
3988                                   container_of(entry, typeof(*resource), entry);
3989
3990         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
3991 }
3992
3993 /**
3994  * Find existing push vlan resource or create and register a new one.
3995  *
3996  * @param [in, out] dev
3997  *   Pointer to rte_eth_dev structure.
3998  * @param[in, out] ref
3999  *   Pointer to port ID action resource reference.
4000  * @parm[in, out] dev_flow
4001  *   Pointer to the dev_flow.
4002  * @param[out] error
4003  *   pointer to error structure.
4004  *
4005  * @return
4006  *   0 on success otherwise -errno and errno is set.
4007  */
4008 static int
4009 flow_dv_push_vlan_action_resource_register
4010                        (struct rte_eth_dev *dev,
4011                         struct mlx5_flow_dv_push_vlan_action_resource *ref,
4012                         struct mlx5_flow *dev_flow,
4013                         struct rte_flow_error *error)
4014 {
4015         struct mlx5_priv *priv = dev->data->dev_private;
4016         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4017         struct mlx5_list_entry *entry;
4018         struct mlx5_flow_cb_ctx ctx = {
4019                 .error = error,
4020                 .data = ref,
4021         };
4022
4023         entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
4024         if (!entry)
4025                 return -rte_errno;
4026         resource = container_of(entry, typeof(*resource), entry);
4027
4028         dev_flow->handle->dvh.rix_push_vlan = resource->idx;
4029         dev_flow->dv.push_vlan_res = resource;
4030         return 0;
4031 }
4032
4033 /**
4034  * Get the size of specific rte_flow_item_type hdr size
4035  *
4036  * @param[in] item_type
4037  *   Tested rte_flow_item_type.
4038  *
4039  * @return
4040  *   sizeof struct item_type, 0 if void or irrelevant.
4041  */
4042 size_t
4043 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
4044 {
4045         size_t retval;
4046
4047         switch (item_type) {
4048         case RTE_FLOW_ITEM_TYPE_ETH:
4049                 retval = sizeof(struct rte_ether_hdr);
4050                 break;
4051         case RTE_FLOW_ITEM_TYPE_VLAN:
4052                 retval = sizeof(struct rte_vlan_hdr);
4053                 break;
4054         case RTE_FLOW_ITEM_TYPE_IPV4:
4055                 retval = sizeof(struct rte_ipv4_hdr);
4056                 break;
4057         case RTE_FLOW_ITEM_TYPE_IPV6:
4058                 retval = sizeof(struct rte_ipv6_hdr);
4059                 break;
4060         case RTE_FLOW_ITEM_TYPE_UDP:
4061                 retval = sizeof(struct rte_udp_hdr);
4062                 break;
4063         case RTE_FLOW_ITEM_TYPE_TCP:
4064                 retval = sizeof(struct rte_tcp_hdr);
4065                 break;
4066         case RTE_FLOW_ITEM_TYPE_VXLAN:
4067         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4068                 retval = sizeof(struct rte_vxlan_hdr);
4069                 break;
4070         case RTE_FLOW_ITEM_TYPE_GRE:
4071         case RTE_FLOW_ITEM_TYPE_NVGRE:
4072                 retval = sizeof(struct rte_gre_hdr);
4073                 break;
4074         case RTE_FLOW_ITEM_TYPE_MPLS:
4075                 retval = sizeof(struct rte_mpls_hdr);
4076                 break;
4077         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
4078         default:
4079                 retval = 0;
4080                 break;
4081         }
4082         return retval;
4083 }
4084
4085 #define MLX5_ENCAP_IPV4_VERSION         0x40
4086 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
4087 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
4088 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
4089 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
4090 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
4091 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
4092
4093 /**
4094  * Convert the encap action data from list of rte_flow_item to raw buffer
4095  *
4096  * @param[in] items
4097  *   Pointer to rte_flow_item objects list.
4098  * @param[out] buf
4099  *   Pointer to the output buffer.
4100  * @param[out] size
4101  *   Pointer to the output buffer size.
4102  * @param[out] error
4103  *   Pointer to the error structure.
4104  *
4105  * @return
4106  *   0 on success, a negative errno value otherwise and rte_errno is set.
4107  */
4108 int
4109 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
4110                            size_t *size, struct rte_flow_error *error)
4111 {
4112         struct rte_ether_hdr *eth = NULL;
4113         struct rte_vlan_hdr *vlan = NULL;
4114         struct rte_ipv4_hdr *ipv4 = NULL;
4115         struct rte_ipv6_hdr *ipv6 = NULL;
4116         struct rte_udp_hdr *udp = NULL;
4117         struct rte_vxlan_hdr *vxlan = NULL;
4118         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
4119         struct rte_gre_hdr *gre = NULL;
4120         size_t len;
4121         size_t temp_size = 0;
4122
4123         if (!items)
4124                 return rte_flow_error_set(error, EINVAL,
4125                                           RTE_FLOW_ERROR_TYPE_ACTION,
4126                                           NULL, "invalid empty data");
4127         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4128                 len = flow_dv_get_item_hdr_len(items->type);
4129                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
4130                         return rte_flow_error_set(error, EINVAL,
4131                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4132                                                   (void *)items->type,
4133                                                   "items total size is too big"
4134                                                   " for encap action");
4135                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
4136                 switch (items->type) {
4137                 case RTE_FLOW_ITEM_TYPE_ETH:
4138                         eth = (struct rte_ether_hdr *)&buf[temp_size];
4139                         break;
4140                 case RTE_FLOW_ITEM_TYPE_VLAN:
4141                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
4142                         if (!eth)
4143                                 return rte_flow_error_set(error, EINVAL,
4144                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4145                                                 (void *)items->type,
4146                                                 "eth header not found");
4147                         if (!eth->ether_type)
4148                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
4149                         break;
4150                 case RTE_FLOW_ITEM_TYPE_IPV4:
4151                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
4152                         if (!vlan && !eth)
4153                                 return rte_flow_error_set(error, EINVAL,
4154                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4155                                                 (void *)items->type,
4156                                                 "neither eth nor vlan"
4157                                                 " header found");
4158                         if (vlan && !vlan->eth_proto)
4159                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4160                         else if (eth && !eth->ether_type)
4161                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4162                         if (!ipv4->version_ihl)
4163                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
4164                                                     MLX5_ENCAP_IPV4_IHL_MIN;
4165                         if (!ipv4->time_to_live)
4166                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
4167                         break;
4168                 case RTE_FLOW_ITEM_TYPE_IPV6:
4169                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
4170                         if (!vlan && !eth)
4171                                 return rte_flow_error_set(error, EINVAL,
4172                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4173                                                 (void *)items->type,
4174                                                 "neither eth nor vlan"
4175                                                 " header found");
4176                         if (vlan && !vlan->eth_proto)
4177                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4178                         else if (eth && !eth->ether_type)
4179                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4180                         if (!ipv6->vtc_flow)
4181                                 ipv6->vtc_flow =
4182                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
4183                         if (!ipv6->hop_limits)
4184                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
4185                         break;
4186                 case RTE_FLOW_ITEM_TYPE_UDP:
4187                         udp = (struct rte_udp_hdr *)&buf[temp_size];
4188                         if (!ipv4 && !ipv6)
4189                                 return rte_flow_error_set(error, EINVAL,
4190                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4191                                                 (void *)items->type,
4192                                                 "ip header not found");
4193                         if (ipv4 && !ipv4->next_proto_id)
4194                                 ipv4->next_proto_id = IPPROTO_UDP;
4195                         else if (ipv6 && !ipv6->proto)
4196                                 ipv6->proto = IPPROTO_UDP;
4197                         break;
4198                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4199                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
4200                         if (!udp)
4201                                 return rte_flow_error_set(error, EINVAL,
4202                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4203                                                 (void *)items->type,
4204                                                 "udp header not found");
4205                         if (!udp->dst_port)
4206                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
4207                         if (!vxlan->vx_flags)
4208                                 vxlan->vx_flags =
4209                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
4210                         break;
4211                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4212                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
4213                         if (!udp)
4214                                 return rte_flow_error_set(error, EINVAL,
4215                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4216                                                 (void *)items->type,
4217                                                 "udp header not found");
4218                         if (!vxlan_gpe->proto)
4219                                 return rte_flow_error_set(error, EINVAL,
4220                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4221                                                 (void *)items->type,
4222                                                 "next protocol not found");
4223                         if (!udp->dst_port)
4224                                 udp->dst_port =
4225                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
4226                         if (!vxlan_gpe->vx_flags)
4227                                 vxlan_gpe->vx_flags =
4228                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
4229                         break;
4230                 case RTE_FLOW_ITEM_TYPE_GRE:
4231                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4232                         gre = (struct rte_gre_hdr *)&buf[temp_size];
4233                         if (!gre->proto)
4234                                 return rte_flow_error_set(error, EINVAL,
4235                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4236                                                 (void *)items->type,
4237                                                 "next protocol not found");
4238                         if (!ipv4 && !ipv6)
4239                                 return rte_flow_error_set(error, EINVAL,
4240                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4241                                                 (void *)items->type,
4242                                                 "ip header not found");
4243                         if (ipv4 && !ipv4->next_proto_id)
4244                                 ipv4->next_proto_id = IPPROTO_GRE;
4245                         else if (ipv6 && !ipv6->proto)
4246                                 ipv6->proto = IPPROTO_GRE;
4247                         break;
4248                 case RTE_FLOW_ITEM_TYPE_VOID:
4249                         break;
4250                 default:
4251                         return rte_flow_error_set(error, EINVAL,
4252                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4253                                                   (void *)items->type,
4254                                                   "unsupported item type");
4255                         break;
4256                 }
4257                 temp_size += len;
4258         }
4259         *size = temp_size;
4260         return 0;
4261 }
4262
4263 static int
4264 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
4265 {
4266         struct rte_ether_hdr *eth = NULL;
4267         struct rte_vlan_hdr *vlan = NULL;
4268         struct rte_ipv6_hdr *ipv6 = NULL;
4269         struct rte_udp_hdr *udp = NULL;
4270         char *next_hdr;
4271         uint16_t proto;
4272
4273         eth = (struct rte_ether_hdr *)data;
4274         next_hdr = (char *)(eth + 1);
4275         proto = RTE_BE16(eth->ether_type);
4276
4277         /* VLAN skipping */
4278         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
4279                 vlan = (struct rte_vlan_hdr *)next_hdr;
4280                 proto = RTE_BE16(vlan->eth_proto);
4281                 next_hdr += sizeof(struct rte_vlan_hdr);
4282         }
4283
4284         /* HW calculates IPv4 csum. no need to proceed */
4285         if (proto == RTE_ETHER_TYPE_IPV4)
4286                 return 0;
4287
4288         /* non IPv4/IPv6 header. not supported */
4289         if (proto != RTE_ETHER_TYPE_IPV6) {
4290                 return rte_flow_error_set(error, ENOTSUP,
4291                                           RTE_FLOW_ERROR_TYPE_ACTION,
4292                                           NULL, "Cannot offload non IPv4/IPv6");
4293         }
4294
4295         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
4296
4297         /* ignore non UDP */
4298         if (ipv6->proto != IPPROTO_UDP)
4299                 return 0;
4300
4301         udp = (struct rte_udp_hdr *)(ipv6 + 1);
4302         udp->dgram_cksum = 0;
4303
4304         return 0;
4305 }
4306
4307 /**
4308  * Convert L2 encap action to DV specification.
4309  *
4310  * @param[in] dev
4311  *   Pointer to rte_eth_dev structure.
4312  * @param[in] action
4313  *   Pointer to action structure.
4314  * @param[in, out] dev_flow
4315  *   Pointer to the mlx5_flow.
4316  * @param[in] transfer
4317  *   Mark if the flow is E-Switch flow.
4318  * @param[out] error
4319  *   Pointer to the error structure.
4320  *
4321  * @return
4322  *   0 on success, a negative errno value otherwise and rte_errno is set.
4323  */
4324 static int
4325 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
4326                                const struct rte_flow_action *action,
4327                                struct mlx5_flow *dev_flow,
4328                                uint8_t transfer,
4329                                struct rte_flow_error *error)
4330 {
4331         const struct rte_flow_item *encap_data;
4332         const struct rte_flow_action_raw_encap *raw_encap_data;
4333         struct mlx5_flow_dv_encap_decap_resource res = {
4334                 .reformat_type =
4335                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
4336                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4337                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
4338         };
4339
4340         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4341                 raw_encap_data =
4342                         (const struct rte_flow_action_raw_encap *)action->conf;
4343                 res.size = raw_encap_data->size;
4344                 memcpy(res.buf, raw_encap_data->data, res.size);
4345         } else {
4346                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
4347                         encap_data =
4348                                 ((const struct rte_flow_action_vxlan_encap *)
4349                                                 action->conf)->definition;
4350                 else
4351                         encap_data =
4352                                 ((const struct rte_flow_action_nvgre_encap *)
4353                                                 action->conf)->definition;
4354                 if (flow_dv_convert_encap_data(encap_data, res.buf,
4355                                                &res.size, error))
4356                         return -rte_errno;
4357         }
4358         if (flow_dv_zero_encap_udp_csum(res.buf, error))
4359                 return -rte_errno;
4360         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4361                 return rte_flow_error_set(error, EINVAL,
4362                                           RTE_FLOW_ERROR_TYPE_ACTION,
4363                                           NULL, "can't create L2 encap action");
4364         return 0;
4365 }
4366
4367 /**
4368  * Convert L2 decap action to DV specification.
4369  *
4370  * @param[in] dev
4371  *   Pointer to rte_eth_dev structure.
4372  * @param[in, out] dev_flow
4373  *   Pointer to the mlx5_flow.
4374  * @param[in] transfer
4375  *   Mark if the flow is E-Switch flow.
4376  * @param[out] error
4377  *   Pointer to the error structure.
4378  *
4379  * @return
4380  *   0 on success, a negative errno value otherwise and rte_errno is set.
4381  */
4382 static int
4383 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
4384                                struct mlx5_flow *dev_flow,
4385                                uint8_t transfer,
4386                                struct rte_flow_error *error)
4387 {
4388         struct mlx5_flow_dv_encap_decap_resource res = {
4389                 .size = 0,
4390                 .reformat_type =
4391                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
4392                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4393                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
4394         };
4395
4396         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4397                 return rte_flow_error_set(error, EINVAL,
4398                                           RTE_FLOW_ERROR_TYPE_ACTION,
4399                                           NULL, "can't create L2 decap action");
4400         return 0;
4401 }
4402
4403 /**
4404  * Convert raw decap/encap (L3 tunnel) action to DV specification.
4405  *
4406  * @param[in] dev
4407  *   Pointer to rte_eth_dev structure.
4408  * @param[in] action
4409  *   Pointer to action structure.
4410  * @param[in, out] dev_flow
4411  *   Pointer to the mlx5_flow.
4412  * @param[in] attr
4413  *   Pointer to the flow attributes.
4414  * @param[out] error
4415  *   Pointer to the error structure.
4416  *
4417  * @return
4418  *   0 on success, a negative errno value otherwise and rte_errno is set.
4419  */
4420 static int
4421 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
4422                                 const struct rte_flow_action *action,
4423                                 struct mlx5_flow *dev_flow,
4424                                 const struct rte_flow_attr *attr,
4425                                 struct rte_flow_error *error)
4426 {
4427         const struct rte_flow_action_raw_encap *encap_data;
4428         struct mlx5_flow_dv_encap_decap_resource res;
4429
4430         memset(&res, 0, sizeof(res));
4431         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
4432         res.size = encap_data->size;
4433         memcpy(res.buf, encap_data->data, res.size);
4434         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
4435                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
4436                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
4437         if (attr->transfer)
4438                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4439         else
4440                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4441                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4442         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4443                 return rte_flow_error_set(error, EINVAL,
4444                                           RTE_FLOW_ERROR_TYPE_ACTION,
4445                                           NULL, "can't create encap action");
4446         return 0;
4447 }
4448
4449 /**
4450  * Create action push VLAN.
4451  *
4452  * @param[in] dev
4453  *   Pointer to rte_eth_dev structure.
4454  * @param[in] attr
4455  *   Pointer to the flow attributes.
4456  * @param[in] vlan
4457  *   Pointer to the vlan to push to the Ethernet header.
4458  * @param[in, out] dev_flow
4459  *   Pointer to the mlx5_flow.
4460  * @param[out] error
4461  *   Pointer to the error structure.
4462  *
4463  * @return
4464  *   0 on success, a negative errno value otherwise and rte_errno is set.
4465  */
4466 static int
4467 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
4468                                 const struct rte_flow_attr *attr,
4469                                 const struct rte_vlan_hdr *vlan,
4470                                 struct mlx5_flow *dev_flow,
4471                                 struct rte_flow_error *error)
4472 {
4473         struct mlx5_flow_dv_push_vlan_action_resource res;
4474
4475         memset(&res, 0, sizeof(res));
4476         res.vlan_tag =
4477                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
4478                                  vlan->vlan_tci);
4479         if (attr->transfer)
4480                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4481         else
4482                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4483                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4484         return flow_dv_push_vlan_action_resource_register
4485                                             (dev, &res, dev_flow, error);
4486 }
4487
4488 /**
4489  * Validate the modify-header actions.
4490  *
4491  * @param[in] action_flags
4492  *   Holds the actions detected until now.
4493  * @param[in] action
4494  *   Pointer to the modify action.
4495  * @param[out] error
4496  *   Pointer to error structure.
4497  *
4498  * @return
4499  *   0 on success, a negative errno value otherwise and rte_errno is set.
4500  */
4501 static int
4502 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
4503                                    const struct rte_flow_action *action,
4504                                    struct rte_flow_error *error)
4505 {
4506         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
4507                 return rte_flow_error_set(error, EINVAL,
4508                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4509                                           NULL, "action configuration not set");
4510         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
4511                 return rte_flow_error_set(error, EINVAL,
4512                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4513                                           "can't have encap action before"
4514                                           " modify action");
4515         return 0;
4516 }
4517
4518 /**
4519  * Validate the modify-header MAC address actions.
4520  *
4521  * @param[in] action_flags
4522  *   Holds the actions detected until now.
4523  * @param[in] action
4524  *   Pointer to the modify action.
4525  * @param[in] item_flags
4526  *   Holds the items detected.
4527  * @param[out] error
4528  *   Pointer to error structure.
4529  *
4530  * @return
4531  *   0 on success, a negative errno value otherwise and rte_errno is set.
4532  */
4533 static int
4534 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
4535                                    const struct rte_flow_action *action,
4536                                    const uint64_t item_flags,
4537                                    struct rte_flow_error *error)
4538 {
4539         int ret = 0;
4540
4541         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4542         if (!ret) {
4543                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
4544                         return rte_flow_error_set(error, EINVAL,
4545                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4546                                                   NULL,
4547                                                   "no L2 item in pattern");
4548         }
4549         return ret;
4550 }
4551
4552 /**
4553  * Validate the modify-header IPv4 address actions.
4554  *
4555  * @param[in] action_flags
4556  *   Holds the actions detected until now.
4557  * @param[in] action
4558  *   Pointer to the modify action.
4559  * @param[in] item_flags
4560  *   Holds the items detected.
4561  * @param[out] error
4562  *   Pointer to error structure.
4563  *
4564  * @return
4565  *   0 on success, a negative errno value otherwise and rte_errno is set.
4566  */
4567 static int
4568 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
4569                                     const struct rte_flow_action *action,
4570                                     const uint64_t item_flags,
4571                                     struct rte_flow_error *error)
4572 {
4573         int ret = 0;
4574         uint64_t layer;
4575
4576         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4577         if (!ret) {
4578                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4579                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4580                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4581                 if (!(item_flags & layer))
4582                         return rte_flow_error_set(error, EINVAL,
4583                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4584                                                   NULL,
4585                                                   "no ipv4 item in pattern");
4586         }
4587         return ret;
4588 }
4589
4590 /**
4591  * Validate the modify-header IPv6 address actions.
4592  *
4593  * @param[in] action_flags
4594  *   Holds the actions detected until now.
4595  * @param[in] action
4596  *   Pointer to the modify action.
4597  * @param[in] item_flags
4598  *   Holds the items detected.
4599  * @param[out] error
4600  *   Pointer to error structure.
4601  *
4602  * @return
4603  *   0 on success, a negative errno value otherwise and rte_errno is set.
4604  */
4605 static int
4606 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
4607                                     const struct rte_flow_action *action,
4608                                     const uint64_t item_flags,
4609                                     struct rte_flow_error *error)
4610 {
4611         int ret = 0;
4612         uint64_t layer;
4613
4614         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4615         if (!ret) {
4616                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4617                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4618                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4619                 if (!(item_flags & layer))
4620                         return rte_flow_error_set(error, EINVAL,
4621                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4622                                                   NULL,
4623                                                   "no ipv6 item in pattern");
4624         }
4625         return ret;
4626 }
4627
4628 /**
4629  * Validate the modify-header TP actions.
4630  *
4631  * @param[in] action_flags
4632  *   Holds the actions detected until now.
4633  * @param[in] action
4634  *   Pointer to the modify action.
4635  * @param[in] item_flags
4636  *   Holds the items detected.
4637  * @param[out] error
4638  *   Pointer to error structure.
4639  *
4640  * @return
4641  *   0 on success, a negative errno value otherwise and rte_errno is set.
4642  */
4643 static int
4644 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
4645                                   const struct rte_flow_action *action,
4646                                   const uint64_t item_flags,
4647                                   struct rte_flow_error *error)
4648 {
4649         int ret = 0;
4650         uint64_t layer;
4651
4652         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4653         if (!ret) {
4654                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4655                                  MLX5_FLOW_LAYER_INNER_L4 :
4656                                  MLX5_FLOW_LAYER_OUTER_L4;
4657                 if (!(item_flags & layer))
4658                         return rte_flow_error_set(error, EINVAL,
4659                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4660                                                   NULL, "no transport layer "
4661                                                   "in pattern");
4662         }
4663         return ret;
4664 }
4665
4666 /**
4667  * Validate the modify-header actions of increment/decrement
4668  * TCP Sequence-number.
4669  *
4670  * @param[in] action_flags
4671  *   Holds the actions detected until now.
4672  * @param[in] action
4673  *   Pointer to the modify action.
4674  * @param[in] item_flags
4675  *   Holds the items detected.
4676  * @param[out] error
4677  *   Pointer to error structure.
4678  *
4679  * @return
4680  *   0 on success, a negative errno value otherwise and rte_errno is set.
4681  */
4682 static int
4683 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
4684                                        const struct rte_flow_action *action,
4685                                        const uint64_t item_flags,
4686                                        struct rte_flow_error *error)
4687 {
4688         int ret = 0;
4689         uint64_t layer;
4690
4691         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4692         if (!ret) {
4693                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4694                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4695                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4696                 if (!(item_flags & layer))
4697                         return rte_flow_error_set(error, EINVAL,
4698                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4699                                                   NULL, "no TCP item in"
4700                                                   " pattern");
4701                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
4702                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
4703                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
4704                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
4705                         return rte_flow_error_set(error, EINVAL,
4706                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4707                                                   NULL,
4708                                                   "cannot decrease and increase"
4709                                                   " TCP sequence number"
4710                                                   " at the same time");
4711         }
4712         return ret;
4713 }
4714
4715 /**
4716  * Validate the modify-header actions of increment/decrement
4717  * TCP Acknowledgment number.
4718  *
4719  * @param[in] action_flags
4720  *   Holds the actions detected until now.
4721  * @param[in] action
4722  *   Pointer to the modify action.
4723  * @param[in] item_flags
4724  *   Holds the items detected.
4725  * @param[out] error
4726  *   Pointer to error structure.
4727  *
4728  * @return
4729  *   0 on success, a negative errno value otherwise and rte_errno is set.
4730  */
4731 static int
4732 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
4733                                        const struct rte_flow_action *action,
4734                                        const uint64_t item_flags,
4735                                        struct rte_flow_error *error)
4736 {
4737         int ret = 0;
4738         uint64_t layer;
4739
4740         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4741         if (!ret) {
4742                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4743                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4744                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4745                 if (!(item_flags & layer))
4746                         return rte_flow_error_set(error, EINVAL,
4747                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4748                                                   NULL, "no TCP item in"
4749                                                   " pattern");
4750                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
4751                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
4752                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
4753                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
4754                         return rte_flow_error_set(error, EINVAL,
4755                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4756                                                   NULL,
4757                                                   "cannot decrease and increase"
4758                                                   " TCP acknowledgment number"
4759                                                   " at the same time");
4760         }
4761         return ret;
4762 }
4763
4764 /**
4765  * Validate the modify-header TTL actions.
4766  *
4767  * @param[in] action_flags
4768  *   Holds the actions detected until now.
4769  * @param[in] action
4770  *   Pointer to the modify action.
4771  * @param[in] item_flags
4772  *   Holds the items detected.
4773  * @param[out] error
4774  *   Pointer to error structure.
4775  *
4776  * @return
4777  *   0 on success, a negative errno value otherwise and rte_errno is set.
4778  */
4779 static int
4780 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
4781                                    const struct rte_flow_action *action,
4782                                    const uint64_t item_flags,
4783                                    struct rte_flow_error *error)
4784 {
4785         int ret = 0;
4786         uint64_t layer;
4787
4788         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4789         if (!ret) {
4790                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4791                                  MLX5_FLOW_LAYER_INNER_L3 :
4792                                  MLX5_FLOW_LAYER_OUTER_L3;
4793                 if (!(item_flags & layer))
4794                         return rte_flow_error_set(error, EINVAL,
4795                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4796                                                   NULL,
4797                                                   "no IP protocol in pattern");
4798         }
4799         return ret;
4800 }
4801
4802 /**
4803  * Validate the generic modify field actions.
4804  * @param[in] dev
4805  *   Pointer to the rte_eth_dev structure.
4806  * @param[in] action_flags
4807  *   Holds the actions detected until now.
4808  * @param[in] action
4809  *   Pointer to the modify action.
4810  * @param[in] attr
4811  *   Pointer to the flow attributes.
4812  * @param[out] error
4813  *   Pointer to error structure.
4814  *
4815  * @return
4816  *   Number of header fields to modify (0 or more) on success,
4817  *   a negative errno value otherwise and rte_errno is set.
4818  */
4819 static int
4820 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev,
4821                                    const uint64_t action_flags,
4822                                    const struct rte_flow_action *action,
4823                                    const struct rte_flow_attr *attr,
4824                                    struct rte_flow_error *error)
4825 {
4826         int ret = 0;
4827         struct mlx5_priv *priv = dev->data->dev_private;
4828         struct mlx5_sh_config *config = &priv->sh->config;
4829         const struct rte_flow_action_modify_field *action_modify_field =
4830                 action->conf;
4831         uint32_t dst_width = mlx5_flow_item_field_width(dev,
4832                                 action_modify_field->dst.field,
4833                                 -1, attr, error);
4834         uint32_t src_width = mlx5_flow_item_field_width(dev,
4835                                 action_modify_field->src.field,
4836                                 dst_width, attr, error);
4837
4838         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4839         if (ret)
4840                 return ret;
4841
4842         if (action_modify_field->width == 0)
4843                 return rte_flow_error_set(error, EINVAL,
4844                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4845                                 "no bits are requested to be modified");
4846         else if (action_modify_field->width > dst_width ||
4847                  action_modify_field->width > src_width)
4848                 return rte_flow_error_set(error, EINVAL,
4849                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4850                                 "cannot modify more bits than"
4851                                 " the width of a field");
4852         if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE &&
4853             action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) {
4854                 if ((action_modify_field->dst.offset +
4855                      action_modify_field->width > dst_width) ||
4856                     (action_modify_field->dst.offset % 32))
4857                         return rte_flow_error_set(error, EINVAL,
4858                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4859                                         "destination offset is too big"
4860                                         " or not aligned to 4 bytes");
4861                 if (action_modify_field->dst.level &&
4862                     action_modify_field->dst.field != RTE_FLOW_FIELD_TAG)
4863                         return rte_flow_error_set(error, ENOTSUP,
4864                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4865                                         "inner header fields modification"
4866                                         " is not supported");
4867         }
4868         if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE &&
4869             action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) {
4870                 if (!attr->transfer && !attr->group)
4871                         return rte_flow_error_set(error, ENOTSUP,
4872                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4873                                         "modify field action is not"
4874                                         " supported for group 0");
4875                 if ((action_modify_field->src.offset +
4876                      action_modify_field->width > src_width) ||
4877                     (action_modify_field->src.offset % 32))
4878                         return rte_flow_error_set(error, EINVAL,
4879                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4880                                         "source offset is too big"
4881                                         " or not aligned to 4 bytes");
4882                 if (action_modify_field->src.level &&
4883                     action_modify_field->src.field != RTE_FLOW_FIELD_TAG)
4884                         return rte_flow_error_set(error, ENOTSUP,
4885                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4886                                         "inner header fields modification"
4887                                         " is not supported");
4888         }
4889         if ((action_modify_field->dst.field ==
4890              action_modify_field->src.field) &&
4891             (action_modify_field->dst.level ==
4892              action_modify_field->src.level))
4893                 return rte_flow_error_set(error, EINVAL,
4894                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4895                                 "source and destination fields"
4896                                 " cannot be the same");
4897         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE ||
4898             action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER ||
4899             action_modify_field->dst.field == RTE_FLOW_FIELD_MARK)
4900                 return rte_flow_error_set(error, EINVAL,
4901                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4902                                 "mark, immediate value or a pointer to it"
4903                                 " cannot be used as a destination");
4904         if (action_modify_field->dst.field == RTE_FLOW_FIELD_START ||
4905             action_modify_field->src.field == RTE_FLOW_FIELD_START)
4906                 return rte_flow_error_set(error, ENOTSUP,
4907                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4908                                 "modifications of an arbitrary"
4909                                 " place in a packet is not supported");
4910         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE ||
4911             action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE)
4912                 return rte_flow_error_set(error, ENOTSUP,
4913                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4914                                 "modifications of the 802.1Q Tag"
4915                                 " Identifier is not supported");
4916         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI ||
4917             action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI)
4918                 return rte_flow_error_set(error, ENOTSUP,
4919                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4920                                 "modifications of the VXLAN Network"
4921                                 " Identifier is not supported");
4922         if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI ||
4923             action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI)
4924                 return rte_flow_error_set(error, ENOTSUP,
4925                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4926                                 "modifications of the GENEVE Network"
4927                                 " Identifier is not supported");
4928         if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK ||
4929             action_modify_field->src.field == RTE_FLOW_FIELD_MARK)
4930                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4931                     !mlx5_flow_ext_mreg_supported(dev))
4932                         return rte_flow_error_set(error, ENOTSUP,
4933                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4934                                         "cannot modify mark in legacy mode"
4935                                         " or without extensive registers");
4936         if (action_modify_field->dst.field == RTE_FLOW_FIELD_META ||
4937             action_modify_field->src.field == RTE_FLOW_FIELD_META) {
4938                 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
4939                     !mlx5_flow_ext_mreg_supported(dev))
4940                         return rte_flow_error_set(error, ENOTSUP,
4941                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4942                                         "cannot modify meta without"
4943                                         " extensive registers support");
4944                 ret = flow_dv_get_metadata_reg(dev, attr, error);
4945                 if (ret < 0 || ret == REG_NON)
4946                         return rte_flow_error_set(error, ENOTSUP,
4947                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4948                                         "cannot modify meta without"
4949                                         " extensive registers available");
4950         }
4951         if (action_modify_field->operation != RTE_FLOW_MODIFY_SET)
4952                 return rte_flow_error_set(error, ENOTSUP,
4953                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4954                                 "add and sub operations"
4955                                 " are not supported");
4956         return (action_modify_field->width / 32) +
4957                !!(action_modify_field->width % 32);
4958 }
4959
4960 /**
4961  * Validate jump action.
4962  *
4963  * @param[in] action
4964  *   Pointer to the jump action.
4965  * @param[in] action_flags
4966  *   Holds the actions detected until now.
4967  * @param[in] attributes
4968  *   Pointer to flow attributes
4969  * @param[in] external
4970  *   Action belongs to flow rule created by request external to PMD.
4971  * @param[out] error
4972  *   Pointer to error structure.
4973  *
4974  * @return
4975  *   0 on success, a negative errno value otherwise and rte_errno is set.
4976  */
4977 static int
4978 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
4979                              const struct mlx5_flow_tunnel *tunnel,
4980                              const struct rte_flow_action *action,
4981                              uint64_t action_flags,
4982                              const struct rte_flow_attr *attributes,
4983                              bool external, struct rte_flow_error *error)
4984 {
4985         uint32_t target_group, table = 0;
4986         int ret = 0;
4987         struct flow_grp_info grp_info = {
4988                 .external = !!external,
4989                 .transfer = !!attributes->transfer,
4990                 .fdb_def_rule = 1,
4991                 .std_tbl_fix = 0
4992         };
4993         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4994                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4995                 return rte_flow_error_set(error, EINVAL,
4996                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4997                                           "can't have 2 fate actions in"
4998                                           " same flow");
4999         if (!action->conf)
5000                 return rte_flow_error_set(error, EINVAL,
5001                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5002                                           NULL, "action configuration not set");
5003         target_group =
5004                 ((const struct rte_flow_action_jump *)action->conf)->group;
5005         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
5006                                        &grp_info, error);
5007         if (ret)
5008                 return ret;
5009         if (attributes->group == target_group &&
5010             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
5011                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
5012                 return rte_flow_error_set(error, EINVAL,
5013                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5014                                           "target group must be other than"
5015                                           " the current flow group");
5016         if (table == 0)
5017                 return rte_flow_error_set(error, EINVAL,
5018                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5019                                           NULL, "root table shouldn't be destination");
5020         return 0;
5021 }
5022
5023 /*
5024  * Validate action PORT_ID / REPRESENTED_PORT.
5025  *
5026  * @param[in] dev
5027  *   Pointer to rte_eth_dev structure.
5028  * @param[in] action_flags
5029  *   Bit-fields that holds the actions detected until now.
5030  * @param[in] action
5031  *   PORT_ID / REPRESENTED_PORT action structure.
5032  * @param[in] attr
5033  *   Attributes of flow that includes this action.
5034  * @param[out] error
5035  *   Pointer to error structure.
5036  *
5037  * @return
5038  *   0 on success, a negative errno value otherwise and rte_errno is set.
5039  */
5040 static int
5041 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
5042                                 uint64_t action_flags,
5043                                 const struct rte_flow_action *action,
5044                                 const struct rte_flow_attr *attr,
5045                                 struct rte_flow_error *error)
5046 {
5047         const struct rte_flow_action_port_id *port_id;
5048         const struct rte_flow_action_ethdev *ethdev;
5049         struct mlx5_priv *act_priv;
5050         struct mlx5_priv *dev_priv;
5051         uint16_t port;
5052
5053         if (!attr->transfer)
5054                 return rte_flow_error_set(error, ENOTSUP,
5055                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5056                                           NULL,
5057                                           "port action is valid in transfer"
5058                                           " mode only");
5059         if (!action || !action->conf)
5060                 return rte_flow_error_set(error, ENOTSUP,
5061                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5062                                           NULL,
5063                                           "port action parameters must be"
5064                                           " specified");
5065         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
5066                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5067                 return rte_flow_error_set(error, EINVAL,
5068                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5069                                           "can have only one fate actions in"
5070                                           " a flow");
5071         dev_priv = mlx5_dev_to_eswitch_info(dev);
5072         if (!dev_priv)
5073                 return rte_flow_error_set(error, rte_errno,
5074                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5075                                           NULL,
5076                                           "failed to obtain E-Switch info");
5077         switch (action->type) {
5078         case RTE_FLOW_ACTION_TYPE_PORT_ID:
5079                 port_id = action->conf;
5080                 port = port_id->original ? dev->data->port_id : port_id->id;
5081                 break;
5082         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
5083                 ethdev = action->conf;
5084                 port = ethdev->port_id;
5085                 break;
5086         default:
5087                 MLX5_ASSERT(false);
5088                 return rte_flow_error_set
5089                                 (error, EINVAL,
5090                                  RTE_FLOW_ERROR_TYPE_ACTION, action,
5091                                  "unknown E-Switch action");
5092         }
5093         act_priv = mlx5_port_to_eswitch_info(port, false);
5094         if (!act_priv)
5095                 return rte_flow_error_set
5096                                 (error, rte_errno,
5097                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, action->conf,
5098                                  "failed to obtain E-Switch port id for port");
5099         if (act_priv->domain_id != dev_priv->domain_id)
5100                 return rte_flow_error_set
5101                                 (error, EINVAL,
5102                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5103                                  "port does not belong to"
5104                                  " E-Switch being configured");
5105         return 0;
5106 }
5107
5108 /**
5109  * Get the maximum number of modify header actions.
5110  *
5111  * @param dev
5112  *   Pointer to rte_eth_dev structure.
5113  * @param root
5114  *   Whether action is on root table.
5115  *
5116  * @return
5117  *   Max number of modify header actions device can support.
5118  */
5119 static inline unsigned int
5120 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
5121                               bool root)
5122 {
5123         /*
5124          * There's no way to directly query the max capacity from FW.
5125          * The maximal value on root table should be assumed to be supported.
5126          */
5127         if (!root)
5128                 return MLX5_MAX_MODIFY_NUM;
5129         else
5130                 return MLX5_ROOT_TBL_MODIFY_NUM;
5131 }
5132
5133 /**
5134  * Validate the meter action.
5135  *
5136  * @param[in] dev
5137  *   Pointer to rte_eth_dev structure.
5138  * @param[in] action_flags
5139  *   Bit-fields that holds the actions detected until now.
5140  * @param[in] item_flags
5141  *   Holds the items detected.
5142  * @param[in] action
5143  *   Pointer to the meter action.
5144  * @param[in] attr
5145  *   Attributes of flow that includes this action.
5146  * @param[in] port_id_item
5147  *   Pointer to item indicating port id.
5148  * @param[out] error
5149  *   Pointer to error structure.
5150  *
5151  * @return
5152  *   0 on success, a negative errno value otherwise and rte_errno is set.
5153  */
5154 static int
5155 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
5156                                 uint64_t action_flags, uint64_t item_flags,
5157                                 const struct rte_flow_action *action,
5158                                 const struct rte_flow_attr *attr,
5159                                 const struct rte_flow_item *port_id_item,
5160                                 bool *def_policy,
5161                                 struct rte_flow_error *error)
5162 {
5163         struct mlx5_priv *priv = dev->data->dev_private;
5164         const struct rte_flow_action_meter *am = action->conf;
5165         struct mlx5_flow_meter_info *fm;
5166         struct mlx5_flow_meter_policy *mtr_policy;
5167         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
5168
5169         if (!am)
5170                 return rte_flow_error_set(error, EINVAL,
5171                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5172                                           "meter action conf is NULL");
5173
5174         if (action_flags & MLX5_FLOW_ACTION_METER)
5175                 return rte_flow_error_set(error, ENOTSUP,
5176                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5177                                           "meter chaining not support");
5178         if (action_flags & MLX5_FLOW_ACTION_JUMP)
5179                 return rte_flow_error_set(error, ENOTSUP,
5180                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5181                                           "meter with jump not support");
5182         if (!priv->mtr_en)
5183                 return rte_flow_error_set(error, ENOTSUP,
5184                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5185                                           NULL,
5186                                           "meter action not supported");
5187         fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL);
5188         if (!fm)
5189                 return rte_flow_error_set(error, EINVAL,
5190                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5191                                           "Meter not found");
5192         /* aso meter can always be shared by different domains */
5193         if (fm->ref_cnt && !priv->sh->meter_aso_en &&
5194             !(fm->transfer == attr->transfer ||
5195               (!fm->ingress && !attr->ingress && attr->egress) ||
5196               (!fm->egress && !attr->egress && attr->ingress)))
5197                 return rte_flow_error_set(error, EINVAL,
5198                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5199                         "Flow attributes domain are either invalid "
5200                         "or have a domain conflict with current "
5201                         "meter attributes");
5202         if (fm->def_policy) {
5203                 if (!((attr->transfer &&
5204                         mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) ||
5205                         (attr->egress &&
5206                         mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) ||
5207                         (attr->ingress &&
5208                         mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS])))
5209                         return rte_flow_error_set(error, EINVAL,
5210                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5211                                           "Flow attributes domain "
5212                                           "have a conflict with current "
5213                                           "meter domain attributes");
5214                 *def_policy = true;
5215         } else {
5216                 mtr_policy = mlx5_flow_meter_policy_find(dev,
5217                                                 fm->policy_id, NULL);
5218                 if (!mtr_policy)
5219                         return rte_flow_error_set(error, EINVAL,
5220                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5221                                           "Invalid policy id for meter ");
5222                 if (!((attr->transfer && mtr_policy->transfer) ||
5223                         (attr->egress && mtr_policy->egress) ||
5224                         (attr->ingress && mtr_policy->ingress)))
5225                         return rte_flow_error_set(error, EINVAL,
5226                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5227                                           "Flow attributes domain "
5228                                           "have a conflict with current "
5229                                           "meter domain attributes");
5230                 if (attr->transfer && mtr_policy->dev) {
5231                         /**
5232                          * When policy has fate action of port_id,
5233                          * the flow should have the same src port as policy.
5234                          */
5235                         struct mlx5_priv *policy_port_priv =
5236                                         mtr_policy->dev->data->dev_private;
5237                         int32_t flow_src_port = priv->representor_id;
5238
5239                         if (port_id_item) {
5240                                 const struct rte_flow_item_port_id *spec =
5241                                                         port_id_item->spec;
5242                                 struct mlx5_priv *port_priv =
5243                                         mlx5_port_to_eswitch_info(spec->id,
5244                                                                   false);
5245                                 if (!port_priv)
5246                                         return rte_flow_error_set(error,
5247                                                 rte_errno,
5248                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5249                                                 spec,
5250                                                 "Failed to get port info.");
5251                                 flow_src_port = port_priv->representor_id;
5252                         }
5253                         if (flow_src_port != policy_port_priv->representor_id)
5254                                 return rte_flow_error_set(error,
5255                                                 rte_errno,
5256                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5257                                                 NULL,
5258                                                 "Flow and meter policy "
5259                                                 "have different src port.");
5260                 } else if (mtr_policy->is_rss) {
5261                         struct mlx5_flow_meter_policy *fp;
5262                         struct mlx5_meter_policy_action_container *acg;
5263                         struct mlx5_meter_policy_action_container *acy;
5264                         const struct rte_flow_action *rss_act;
5265                         int ret;
5266
5267                         fp = mlx5_flow_meter_hierarchy_get_final_policy(dev,
5268                                                                 mtr_policy);
5269                         if (fp == NULL)
5270                                 return rte_flow_error_set(error, EINVAL,
5271                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5272                                                   "Unable to get the final "
5273                                                   "policy in the hierarchy");
5274                         acg = &fp->act_cnt[RTE_COLOR_GREEN];
5275                         acy = &fp->act_cnt[RTE_COLOR_YELLOW];
5276                         MLX5_ASSERT(acg->fate_action ==
5277                                     MLX5_FLOW_FATE_SHARED_RSS ||
5278                                     acy->fate_action ==
5279                                     MLX5_FLOW_FATE_SHARED_RSS);
5280                         if (acg->fate_action == MLX5_FLOW_FATE_SHARED_RSS)
5281                                 rss_act = acg->rss;
5282                         else
5283                                 rss_act = acy->rss;
5284                         ret = mlx5_flow_validate_action_rss(rss_act,
5285                                         action_flags, dev, attr,
5286                                         item_flags, error);
5287                         if (ret)
5288                                 return ret;
5289                 }
5290                 *def_policy = false;
5291         }
5292         return 0;
5293 }
5294
5295 /**
5296  * Validate the age action.
5297  *
5298  * @param[in] action_flags
5299  *   Holds the actions detected until now.
5300  * @param[in] action
5301  *   Pointer to the age action.
5302  * @param[in] dev
5303  *   Pointer to the Ethernet device structure.
5304  * @param[out] error
5305  *   Pointer to error structure.
5306  *
5307  * @return
5308  *   0 on success, a negative errno value otherwise and rte_errno is set.
5309  */
5310 static int
5311 flow_dv_validate_action_age(uint64_t action_flags,
5312                             const struct rte_flow_action *action,
5313                             struct rte_eth_dev *dev,
5314                             struct rte_flow_error *error)
5315 {
5316         struct mlx5_priv *priv = dev->data->dev_private;
5317         const struct rte_flow_action_age *age = action->conf;
5318
5319         if (!priv->sh->cdev->config.devx ||
5320             (priv->sh->cmng.counter_fallback && !priv->sh->aso_age_mng))
5321                 return rte_flow_error_set(error, ENOTSUP,
5322                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5323                                           NULL,
5324                                           "age action not supported");
5325         if (!(action->conf))
5326                 return rte_flow_error_set(error, EINVAL,
5327                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5328                                           "configuration cannot be null");
5329         if (!(age->timeout))
5330                 return rte_flow_error_set(error, EINVAL,
5331                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5332                                           "invalid timeout value 0");
5333         if (action_flags & MLX5_FLOW_ACTION_AGE)
5334                 return rte_flow_error_set(error, EINVAL,
5335                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5336                                           "duplicate age actions set");
5337         return 0;
5338 }
5339
5340 /**
5341  * Validate the modify-header IPv4 DSCP actions.
5342  *
5343  * @param[in] action_flags
5344  *   Holds the actions detected until now.
5345  * @param[in] action
5346  *   Pointer to the modify action.
5347  * @param[in] item_flags
5348  *   Holds the items detected.
5349  * @param[out] error
5350  *   Pointer to error structure.
5351  *
5352  * @return
5353  *   0 on success, a negative errno value otherwise and rte_errno is set.
5354  */
5355 static int
5356 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
5357                                          const struct rte_flow_action *action,
5358                                          const uint64_t item_flags,
5359                                          struct rte_flow_error *error)
5360 {
5361         int ret = 0;
5362
5363         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5364         if (!ret) {
5365                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
5366                         return rte_flow_error_set(error, EINVAL,
5367                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5368                                                   NULL,
5369                                                   "no ipv4 item in pattern");
5370         }
5371         return ret;
5372 }
5373
5374 /**
5375  * Validate the modify-header IPv6 DSCP actions.
5376  *
5377  * @param[in] action_flags
5378  *   Holds the actions detected until now.
5379  * @param[in] action
5380  *   Pointer to the modify action.
5381  * @param[in] item_flags
5382  *   Holds the items detected.
5383  * @param[out] error
5384  *   Pointer to error structure.
5385  *
5386  * @return
5387  *   0 on success, a negative errno value otherwise and rte_errno is set.
5388  */
5389 static int
5390 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
5391                                          const struct rte_flow_action *action,
5392                                          const uint64_t item_flags,
5393                                          struct rte_flow_error *error)
5394 {
5395         int ret = 0;
5396
5397         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5398         if (!ret) {
5399                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
5400                         return rte_flow_error_set(error, EINVAL,
5401                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5402                                                   NULL,
5403                                                   "no ipv6 item in pattern");
5404         }
5405         return ret;
5406 }
5407
5408 int
5409 flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
5410                         struct mlx5_list_entry *entry, void *cb_ctx)
5411 {
5412         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5413         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5414         struct mlx5_flow_dv_modify_hdr_resource *resource =
5415                                   container_of(entry, typeof(*resource), entry);
5416         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5417
5418         key_len += ref->actions_num * sizeof(ref->actions[0]);
5419         return ref->actions_num != resource->actions_num ||
5420                memcmp(&ref->ft_type, &resource->ft_type, key_len);
5421 }
5422
5423 static struct mlx5_indexed_pool *
5424 flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
5425 {
5426         struct mlx5_indexed_pool *ipool = __atomic_load_n
5427                                      (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
5428
5429         if (!ipool) {
5430                 struct mlx5_indexed_pool *expected = NULL;
5431                 struct mlx5_indexed_pool_config cfg =
5432                     (struct mlx5_indexed_pool_config) {
5433                        .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
5434                                                                    (index + 1) *
5435                                            sizeof(struct mlx5_modification_cmd),
5436                        .trunk_size = 64,
5437                        .grow_trunk = 3,
5438                        .grow_shift = 2,
5439                        .need_lock = 1,
5440                        .release_mem_en = !!sh->config.reclaim_mode,
5441                        .per_core_cache =
5442                                        sh->config.reclaim_mode ? 0 : (1 << 16),
5443                        .malloc = mlx5_malloc,
5444                        .free = mlx5_free,
5445                        .type = "mlx5_modify_action_resource",
5446                 };
5447
5448                 cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
5449                 ipool = mlx5_ipool_create(&cfg);
5450                 if (!ipool)
5451                         return NULL;
5452                 if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
5453                                                  &expected, ipool, false,
5454                                                  __ATOMIC_SEQ_CST,
5455                                                  __ATOMIC_SEQ_CST)) {
5456                         mlx5_ipool_destroy(ipool);
5457                         ipool = __atomic_load_n(&sh->mdh_ipools[index],
5458                                                 __ATOMIC_SEQ_CST);
5459                 }
5460         }
5461         return ipool;
5462 }
5463
5464 struct mlx5_list_entry *
5465 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
5466 {
5467         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5468         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5469         struct mlx5dv_dr_domain *ns;
5470         struct mlx5_flow_dv_modify_hdr_resource *entry;
5471         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5472         struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
5473                                                           ref->actions_num - 1);
5474         int ret;
5475         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5476         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5477         uint32_t idx;
5478
5479         if (unlikely(!ipool)) {
5480                 rte_flow_error_set(ctx->error, ENOMEM,
5481                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5482                                    NULL, "cannot allocate modify ipool");
5483                 return NULL;
5484         }
5485         entry = mlx5_ipool_zmalloc(ipool, &idx);
5486         if (!entry) {
5487                 rte_flow_error_set(ctx->error, ENOMEM,
5488                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5489                                    "cannot allocate resource memory");
5490                 return NULL;
5491         }
5492         rte_memcpy(&entry->ft_type,
5493                    RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
5494                    key_len + data_len);
5495         if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
5496                 ns = sh->fdb_domain;
5497         else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
5498                 ns = sh->tx_domain;
5499         else
5500                 ns = sh->rx_domain;
5501         ret = mlx5_flow_os_create_flow_action_modify_header
5502                                         (sh->cdev->ctx, ns, entry,
5503                                          data_len, &entry->action);
5504         if (ret) {
5505                 mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
5506                 rte_flow_error_set(ctx->error, ENOMEM,
5507                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5508                                    NULL, "cannot create modification action");
5509                 return NULL;
5510         }
5511         entry->idx = idx;
5512         return &entry->entry;
5513 }
5514
5515 struct mlx5_list_entry *
5516 flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
5517                         void *cb_ctx)
5518 {
5519         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5520         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5521         struct mlx5_flow_dv_modify_hdr_resource *entry;
5522         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5523         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5524         uint32_t idx;
5525
5526         entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
5527                                   &idx);
5528         if (!entry) {
5529                 rte_flow_error_set(ctx->error, ENOMEM,
5530                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5531                                    "cannot allocate resource memory");
5532                 return NULL;
5533         }
5534         memcpy(entry, oentry, sizeof(*entry) + data_len);
5535         entry->idx = idx;
5536         return &entry->entry;
5537 }
5538
5539 void
5540 flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
5541 {
5542         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5543         struct mlx5_flow_dv_modify_hdr_resource *res =
5544                 container_of(entry, typeof(*res), entry);
5545
5546         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
5547 }
5548
5549 /**
5550  * Validate the sample action.
5551  *
5552  * @param[in, out] action_flags
5553  *   Holds the actions detected until now.
5554  * @param[in] action
5555  *   Pointer to the sample action.
5556  * @param[in] dev
5557  *   Pointer to the Ethernet device structure.
5558  * @param[in] attr
5559  *   Attributes of flow that includes this action.
5560  * @param[in] item_flags
5561  *   Holds the items detected.
5562  * @param[in] rss
5563  *   Pointer to the RSS action.
5564  * @param[out] sample_rss
5565  *   Pointer to the RSS action in sample action list.
5566  * @param[out] count
5567  *   Pointer to the COUNT action in sample action list.
5568  * @param[out] fdb_mirror_limit
5569  *   Pointer to the FDB mirror limitation flag.
5570  * @param[out] error
5571  *   Pointer to error structure.
5572  *
5573  * @return
5574  *   0 on success, a negative errno value otherwise and rte_errno is set.
5575  */
5576 static int
5577 flow_dv_validate_action_sample(uint64_t *action_flags,
5578                                const struct rte_flow_action *action,
5579                                struct rte_eth_dev *dev,
5580                                const struct rte_flow_attr *attr,
5581                                uint64_t item_flags,
5582                                const struct rte_flow_action_rss *rss,
5583                                const struct rte_flow_action_rss **sample_rss,
5584                                const struct rte_flow_action_count **count,
5585                                int *fdb_mirror_limit,
5586                                struct rte_flow_error *error)
5587 {
5588         struct mlx5_priv *priv = dev->data->dev_private;
5589         struct mlx5_sh_config *dev_conf = &priv->sh->config;
5590         const struct rte_flow_action_sample *sample = action->conf;
5591         const struct rte_flow_action *act;
5592         uint64_t sub_action_flags = 0;
5593         uint16_t queue_index = 0xFFFF;
5594         int actions_n = 0;
5595         int ret;
5596
5597         if (!sample)
5598                 return rte_flow_error_set(error, EINVAL,
5599                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5600                                           "configuration cannot be NULL");
5601         if (sample->ratio == 0)
5602                 return rte_flow_error_set(error, EINVAL,
5603                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5604                                           "ratio value starts from 1");
5605         if (!priv->sh->cdev->config.devx ||
5606             (sample->ratio > 0 && !priv->sampler_en))
5607                 return rte_flow_error_set(error, ENOTSUP,
5608                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5609                                           NULL,
5610                                           "sample action not supported");
5611         if (*action_flags & MLX5_FLOW_ACTION_SAMPLE)
5612                 return rte_flow_error_set(error, EINVAL,
5613                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5614                                           "Multiple sample actions not "
5615                                           "supported");
5616         if (*action_flags & MLX5_FLOW_ACTION_METER)
5617                 return rte_flow_error_set(error, EINVAL,
5618                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5619                                           "wrong action order, meter should "
5620                                           "be after sample action");
5621         if (*action_flags & MLX5_FLOW_ACTION_JUMP)
5622                 return rte_flow_error_set(error, EINVAL,
5623                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5624                                           "wrong action order, jump should "
5625                                           "be after sample action");
5626         if (*action_flags & MLX5_FLOW_ACTION_CT)
5627                 return rte_flow_error_set(error, EINVAL,
5628                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5629                                           "Sample after CT not supported");
5630         act = sample->actions;
5631         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
5632                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5633                         return rte_flow_error_set(error, ENOTSUP,
5634                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5635                                                   act, "too many actions");
5636                 switch (act->type) {
5637                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5638                         ret = mlx5_flow_validate_action_queue(act,
5639                                                               sub_action_flags,
5640                                                               dev,
5641                                                               attr, error);
5642                         if (ret < 0)
5643                                 return ret;
5644                         queue_index = ((const struct rte_flow_action_queue *)
5645                                                         (act->conf))->index;
5646                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
5647                         ++actions_n;
5648                         break;
5649                 case RTE_FLOW_ACTION_TYPE_RSS:
5650                         *sample_rss = act->conf;
5651                         ret = mlx5_flow_validate_action_rss(act,
5652                                                             sub_action_flags,
5653                                                             dev, attr,
5654                                                             item_flags,
5655                                                             error);
5656                         if (ret < 0)
5657                                 return ret;
5658                         if (rss && *sample_rss &&
5659                             ((*sample_rss)->level != rss->level ||
5660                             (*sample_rss)->types != rss->types))
5661                                 return rte_flow_error_set(error, ENOTSUP,
5662                                         RTE_FLOW_ERROR_TYPE_ACTION,
5663                                         NULL,
5664                                         "Can't use the different RSS types "
5665                                         "or level in the same flow");
5666                         if (*sample_rss != NULL && (*sample_rss)->queue_num)
5667                                 queue_index = (*sample_rss)->queue[0];
5668                         sub_action_flags |= MLX5_FLOW_ACTION_RSS;
5669                         ++actions_n;
5670                         break;
5671                 case RTE_FLOW_ACTION_TYPE_MARK:
5672                         ret = flow_dv_validate_action_mark(dev, act,
5673                                                            sub_action_flags,
5674                                                            attr, error);
5675                         if (ret < 0)
5676                                 return ret;
5677                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
5678                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
5679                                                 MLX5_FLOW_ACTION_MARK_EXT;
5680                         else
5681                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
5682                         ++actions_n;
5683                         break;
5684                 case RTE_FLOW_ACTION_TYPE_COUNT:
5685                         ret = flow_dv_validate_action_count
5686                                 (dev, false, *action_flags | sub_action_flags,
5687                                  attr, error);
5688                         if (ret < 0)
5689                                 return ret;
5690                         *count = act->conf;
5691                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
5692                         *action_flags |= MLX5_FLOW_ACTION_COUNT;
5693                         ++actions_n;
5694                         break;
5695                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5696                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
5697                         ret = flow_dv_validate_action_port_id(dev,
5698                                                               sub_action_flags,
5699                                                               act,
5700                                                               attr,
5701                                                               error);
5702                         if (ret)
5703                                 return ret;
5704                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5705                         ++actions_n;
5706                         break;
5707                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5708                         ret = flow_dv_validate_action_raw_encap_decap
5709                                 (dev, NULL, act->conf, attr, &sub_action_flags,
5710                                  &actions_n, action, item_flags, error);
5711                         if (ret < 0)
5712                                 return ret;
5713                         ++actions_n;
5714                         break;
5715                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5716                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5717                         ret = flow_dv_validate_action_l2_encap(dev,
5718                                                                sub_action_flags,
5719                                                                act, attr,
5720                                                                error);
5721                         if (ret < 0)
5722                                 return ret;
5723                         sub_action_flags |= MLX5_FLOW_ACTION_ENCAP;
5724                         ++actions_n;
5725                         break;
5726                 default:
5727                         return rte_flow_error_set(error, ENOTSUP,
5728                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5729                                                   NULL,
5730                                                   "Doesn't support optional "
5731                                                   "action");
5732                 }
5733         }
5734         if (attr->ingress && !attr->transfer) {
5735                 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE |
5736                                           MLX5_FLOW_ACTION_RSS)))
5737                         return rte_flow_error_set(error, EINVAL,
5738                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5739                                                   NULL,
5740                                                   "Ingress must has a dest "
5741                                                   "QUEUE for Sample");
5742         } else if (attr->egress && !attr->transfer) {
5743                 return rte_flow_error_set(error, ENOTSUP,
5744                                           RTE_FLOW_ERROR_TYPE_ACTION,
5745                                           NULL,
5746                                           "Sample Only support Ingress "
5747                                           "or E-Switch");
5748         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
5749                 MLX5_ASSERT(attr->transfer);
5750                 if (sample->ratio > 1)
5751                         return rte_flow_error_set(error, ENOTSUP,
5752                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5753                                                   NULL,
5754                                                   "E-Switch doesn't support "
5755                                                   "any optional action "
5756                                                   "for sampling");
5757                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
5758                         return rte_flow_error_set(error, ENOTSUP,
5759                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5760                                                   NULL,
5761                                                   "unsupported action QUEUE");
5762                 if (sub_action_flags & MLX5_FLOW_ACTION_RSS)
5763                         return rte_flow_error_set(error, ENOTSUP,
5764                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5765                                                   NULL,
5766                                                   "unsupported action QUEUE");
5767                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
5768                         return rte_flow_error_set(error, EINVAL,
5769                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5770                                                   NULL,
5771                                                   "E-Switch must has a dest "
5772                                                   "port for mirroring");
5773                 if (!priv->sh->cdev->config.hca_attr.reg_c_preserve &&
5774                      priv->representor_id != UINT16_MAX)
5775                         *fdb_mirror_limit = 1;
5776         }
5777         /* Continue validation for Xcap actions.*/
5778         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
5779             (queue_index == 0xFFFF || !mlx5_rxq_is_hairpin(dev, queue_index))) {
5780                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5781                      MLX5_FLOW_XCAP_ACTIONS)
5782                         return rte_flow_error_set(error, ENOTSUP,
5783                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5784                                                   NULL, "encap and decap "
5785                                                   "combination aren't "
5786                                                   "supported");
5787                 if (!attr->transfer && attr->ingress && (sub_action_flags &
5788                                                         MLX5_FLOW_ACTION_ENCAP))
5789                         return rte_flow_error_set(error, ENOTSUP,
5790                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5791                                                   NULL, "encap is not supported"
5792                                                   " for ingress traffic");
5793         }
5794         return 0;
5795 }
5796
5797 /**
5798  * Find existing modify-header resource or create and register a new one.
5799  *
5800  * @param dev[in, out]
5801  *   Pointer to rte_eth_dev structure.
5802  * @param[in, out] resource
5803  *   Pointer to modify-header resource.
5804  * @parm[in, out] dev_flow
5805  *   Pointer to the dev_flow.
5806  * @param[out] error
5807  *   pointer to error structure.
5808  *
5809  * @return
5810  *   0 on success otherwise -errno and errno is set.
5811  */
5812 static int
5813 flow_dv_modify_hdr_resource_register
5814                         (struct rte_eth_dev *dev,
5815                          struct mlx5_flow_dv_modify_hdr_resource *resource,
5816                          struct mlx5_flow *dev_flow,
5817                          struct rte_flow_error *error)
5818 {
5819         struct mlx5_priv *priv = dev->data->dev_private;
5820         struct mlx5_dev_ctx_shared *sh = priv->sh;
5821         uint32_t key_len = sizeof(*resource) -
5822                            offsetof(typeof(*resource), ft_type) +
5823                            resource->actions_num * sizeof(resource->actions[0]);
5824         struct mlx5_list_entry *entry;
5825         struct mlx5_flow_cb_ctx ctx = {
5826                 .error = error,
5827                 .data = resource,
5828         };
5829         struct mlx5_hlist *modify_cmds;
5830         uint64_t key64;
5831
5832         modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
5833                                 "hdr_modify",
5834                                 MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
5835                                 true, false, sh,
5836                                 flow_dv_modify_create_cb,
5837                                 flow_dv_modify_match_cb,
5838                                 flow_dv_modify_remove_cb,
5839                                 flow_dv_modify_clone_cb,
5840                                 flow_dv_modify_clone_free_cb,
5841                                 error);
5842         if (unlikely(!modify_cmds))
5843                 return -rte_errno;
5844         resource->root = !dev_flow->dv.group;
5845         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
5846                                                                 resource->root))
5847                 return rte_flow_error_set(error, EOVERFLOW,
5848                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5849                                           "too many modify header items");
5850         key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
5851         entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
5852         if (!entry)
5853                 return -rte_errno;
5854         resource = container_of(entry, typeof(*resource), entry);
5855         dev_flow->handle->dvh.modify_hdr = resource;
5856         return 0;
5857 }
5858
5859 /**
5860  * Get DV flow counter by index.
5861  *
5862  * @param[in] dev
5863  *   Pointer to the Ethernet device structure.
5864  * @param[in] idx
5865  *   mlx5 flow counter index in the container.
5866  * @param[out] ppool
5867  *   mlx5 flow counter pool in the container.
5868  *
5869  * @return
5870  *   Pointer to the counter, NULL otherwise.
5871  */
5872 static struct mlx5_flow_counter *
5873 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
5874                            uint32_t idx,
5875                            struct mlx5_flow_counter_pool **ppool)
5876 {
5877         struct mlx5_priv *priv = dev->data->dev_private;
5878         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5879         struct mlx5_flow_counter_pool *pool;
5880
5881         /* Decrease to original index and clear shared bit. */
5882         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
5883         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
5884         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
5885         MLX5_ASSERT(pool);
5886         if (ppool)
5887                 *ppool = pool;
5888         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
5889 }
5890
5891 /**
5892  * Check the devx counter belongs to the pool.
5893  *
5894  * @param[in] pool
5895  *   Pointer to the counter pool.
5896  * @param[in] id
5897  *   The counter devx ID.
5898  *
5899  * @return
5900  *   True if counter belongs to the pool, false otherwise.
5901  */
5902 static bool
5903 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
5904 {
5905         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
5906                    MLX5_COUNTERS_PER_POOL;
5907
5908         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
5909                 return true;
5910         return false;
5911 }
5912
5913 /**
5914  * Get a pool by devx counter ID.
5915  *
5916  * @param[in] cmng
5917  *   Pointer to the counter management.
5918  * @param[in] id
5919  *   The counter devx ID.
5920  *
5921  * @return
5922  *   The counter pool pointer if exists, NULL otherwise,
5923  */
5924 static struct mlx5_flow_counter_pool *
5925 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
5926 {
5927         uint32_t i;
5928         struct mlx5_flow_counter_pool *pool = NULL;
5929
5930         rte_spinlock_lock(&cmng->pool_update_sl);
5931         /* Check last used pool. */
5932         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
5933             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
5934                 pool = cmng->pools[cmng->last_pool_idx];
5935                 goto out;
5936         }
5937         /* ID out of range means no suitable pool in the container. */
5938         if (id > cmng->max_id || id < cmng->min_id)
5939                 goto out;
5940         /*
5941          * Find the pool from the end of the container, since mostly counter
5942          * ID is sequence increasing, and the last pool should be the needed
5943          * one.
5944          */
5945         i = cmng->n_valid;
5946         while (i--) {
5947                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
5948
5949                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
5950                         pool = pool_tmp;
5951                         break;
5952                 }
5953         }
5954 out:
5955         rte_spinlock_unlock(&cmng->pool_update_sl);
5956         return pool;
5957 }
5958
5959 /**
5960  * Resize a counter container.
5961  *
5962  * @param[in] dev
5963  *   Pointer to the Ethernet device structure.
5964  *
5965  * @return
5966  *   0 on success, otherwise negative errno value and rte_errno is set.
5967  */
5968 static int
5969 flow_dv_container_resize(struct rte_eth_dev *dev)
5970 {
5971         struct mlx5_priv *priv = dev->data->dev_private;
5972         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5973         void *old_pools = cmng->pools;
5974         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
5975         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
5976         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
5977
5978         if (!pools) {
5979                 rte_errno = ENOMEM;
5980                 return -ENOMEM;
5981         }
5982         if (old_pools)
5983                 memcpy(pools, old_pools, cmng->n *
5984                                        sizeof(struct mlx5_flow_counter_pool *));
5985         cmng->n = resize;
5986         cmng->pools = pools;
5987         if (old_pools)
5988                 mlx5_free(old_pools);
5989         return 0;
5990 }
5991
5992 /**
5993  * Query a devx flow counter.
5994  *
5995  * @param[in] dev
5996  *   Pointer to the Ethernet device structure.
5997  * @param[in] counter
5998  *   Index to the flow counter.
5999  * @param[out] pkts
6000  *   The statistics value of packets.
6001  * @param[out] bytes
6002  *   The statistics value of bytes.
6003  *
6004  * @return
6005  *   0 on success, otherwise a negative errno value and rte_errno is set.
6006  */
6007 static inline int
6008 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
6009                      uint64_t *bytes)
6010 {
6011         struct mlx5_priv *priv = dev->data->dev_private;
6012         struct mlx5_flow_counter_pool *pool = NULL;
6013         struct mlx5_flow_counter *cnt;
6014         int offset;
6015
6016         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6017         MLX5_ASSERT(pool);
6018         if (priv->sh->cmng.counter_fallback)
6019                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
6020                                         0, pkts, bytes, 0, NULL, NULL, 0);
6021         rte_spinlock_lock(&pool->sl);
6022         if (!pool->raw) {
6023                 *pkts = 0;
6024                 *bytes = 0;
6025         } else {
6026                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
6027                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
6028                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
6029         }
6030         rte_spinlock_unlock(&pool->sl);
6031         return 0;
6032 }
6033
6034 /**
6035  * Create and initialize a new counter pool.
6036  *
6037  * @param[in] dev
6038  *   Pointer to the Ethernet device structure.
6039  * @param[out] dcs
6040  *   The devX counter handle.
6041  * @param[in] age
6042  *   Whether the pool is for counter that was allocated for aging.
6043  * @param[in/out] cont_cur
6044  *   Pointer to the container pointer, it will be update in pool resize.
6045  *
6046  * @return
6047  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
6048  */
6049 static struct mlx5_flow_counter_pool *
6050 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
6051                     uint32_t age)
6052 {
6053         struct mlx5_priv *priv = dev->data->dev_private;
6054         struct mlx5_flow_counter_pool *pool;
6055         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6056         bool fallback = priv->sh->cmng.counter_fallback;
6057         uint32_t size = sizeof(*pool);
6058
6059         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
6060         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
6061         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
6062         if (!pool) {
6063                 rte_errno = ENOMEM;
6064                 return NULL;
6065         }
6066         pool->raw = NULL;
6067         pool->is_aged = !!age;
6068         pool->query_gen = 0;
6069         pool->min_dcs = dcs;
6070         rte_spinlock_init(&pool->sl);
6071         rte_spinlock_init(&pool->csl);
6072         TAILQ_INIT(&pool->counters[0]);
6073         TAILQ_INIT(&pool->counters[1]);
6074         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
6075         rte_spinlock_lock(&cmng->pool_update_sl);
6076         pool->index = cmng->n_valid;
6077         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
6078                 mlx5_free(pool);
6079                 rte_spinlock_unlock(&cmng->pool_update_sl);
6080                 return NULL;
6081         }
6082         cmng->pools[pool->index] = pool;
6083         cmng->n_valid++;
6084         if (unlikely(fallback)) {
6085                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
6086
6087                 if (base < cmng->min_id)
6088                         cmng->min_id = base;
6089                 if (base > cmng->max_id)
6090                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
6091                 cmng->last_pool_idx = pool->index;
6092         }
6093         rte_spinlock_unlock(&cmng->pool_update_sl);
6094         return pool;
6095 }
6096
6097 /**
6098  * Prepare a new counter and/or a new counter pool.
6099  *
6100  * @param[in] dev
6101  *   Pointer to the Ethernet device structure.
6102  * @param[out] cnt_free
6103  *   Where to put the pointer of a new counter.
6104  * @param[in] age
6105  *   Whether the pool is for counter that was allocated for aging.
6106  *
6107  * @return
6108  *   The counter pool pointer and @p cnt_free is set on success,
6109  *   NULL otherwise and rte_errno is set.
6110  */
6111 static struct mlx5_flow_counter_pool *
6112 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
6113                              struct mlx5_flow_counter **cnt_free,
6114                              uint32_t age)
6115 {
6116         struct mlx5_priv *priv = dev->data->dev_private;
6117         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6118         struct mlx5_flow_counter_pool *pool;
6119         struct mlx5_counters tmp_tq;
6120         struct mlx5_devx_obj *dcs = NULL;
6121         struct mlx5_flow_counter *cnt;
6122         enum mlx5_counter_type cnt_type =
6123                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6124         bool fallback = priv->sh->cmng.counter_fallback;
6125         uint32_t i;
6126
6127         if (fallback) {
6128                 /* bulk_bitmap must be 0 for single counter allocation. */
6129                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0);
6130                 if (!dcs)
6131                         return NULL;
6132                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
6133                 if (!pool) {
6134                         pool = flow_dv_pool_create(dev, dcs, age);
6135                         if (!pool) {
6136                                 mlx5_devx_cmd_destroy(dcs);
6137                                 return NULL;
6138                         }
6139                 }
6140                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
6141                 cnt = MLX5_POOL_GET_CNT(pool, i);
6142                 cnt->pool = pool;
6143                 cnt->dcs_when_free = dcs;
6144                 *cnt_free = cnt;
6145                 return pool;
6146         }
6147         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0x4);
6148         if (!dcs) {
6149                 rte_errno = ENODATA;
6150                 return NULL;
6151         }
6152         pool = flow_dv_pool_create(dev, dcs, age);
6153         if (!pool) {
6154                 mlx5_devx_cmd_destroy(dcs);
6155                 return NULL;
6156         }
6157         TAILQ_INIT(&tmp_tq);
6158         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
6159                 cnt = MLX5_POOL_GET_CNT(pool, i);
6160                 cnt->pool = pool;
6161                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
6162         }
6163         rte_spinlock_lock(&cmng->csl[cnt_type]);
6164         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
6165         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6166         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
6167         (*cnt_free)->pool = pool;
6168         return pool;
6169 }
6170
6171 /**
6172  * Allocate a flow counter.
6173  *
6174  * @param[in] dev
6175  *   Pointer to the Ethernet device structure.
6176  * @param[in] age
6177  *   Whether the counter was allocated for aging.
6178  *
6179  * @return
6180  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6181  */
6182 static uint32_t
6183 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
6184 {
6185         struct mlx5_priv *priv = dev->data->dev_private;
6186         struct mlx5_flow_counter_pool *pool = NULL;
6187         struct mlx5_flow_counter *cnt_free = NULL;
6188         bool fallback = priv->sh->cmng.counter_fallback;
6189         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6190         enum mlx5_counter_type cnt_type =
6191                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6192         uint32_t cnt_idx;
6193
6194         if (!priv->sh->cdev->config.devx) {
6195                 rte_errno = ENOTSUP;
6196                 return 0;
6197         }
6198         /* Get free counters from container. */
6199         rte_spinlock_lock(&cmng->csl[cnt_type]);
6200         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
6201         if (cnt_free)
6202                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
6203         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6204         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
6205                 goto err;
6206         pool = cnt_free->pool;
6207         if (fallback)
6208                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
6209         /* Create a DV counter action only in the first time usage. */
6210         if (!cnt_free->action) {
6211                 uint16_t offset;
6212                 struct mlx5_devx_obj *dcs;
6213                 int ret;
6214
6215                 if (!fallback) {
6216                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
6217                         dcs = pool->min_dcs;
6218                 } else {
6219                         offset = 0;
6220                         dcs = cnt_free->dcs_when_free;
6221                 }
6222                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
6223                                                             &cnt_free->action);
6224                 if (ret) {
6225                         rte_errno = errno;
6226                         goto err;
6227                 }
6228         }
6229         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
6230                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
6231         /* Update the counter reset values. */
6232         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
6233                                  &cnt_free->bytes))
6234                 goto err;
6235         if (!fallback && !priv->sh->cmng.query_thread_on)
6236                 /* Start the asynchronous batch query by the host thread. */
6237                 mlx5_set_query_alarm(priv->sh);
6238         /*
6239          * When the count action isn't shared (by ID), shared_info field is
6240          * used for indirect action API's refcnt.
6241          * When the counter action is not shared neither by ID nor by indirect
6242          * action API, shared info must be 1.
6243          */
6244         cnt_free->shared_info.refcnt = 1;
6245         return cnt_idx;
6246 err:
6247         if (cnt_free) {
6248                 cnt_free->pool = pool;
6249                 if (fallback)
6250                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
6251                 rte_spinlock_lock(&cmng->csl[cnt_type]);
6252                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
6253                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
6254         }
6255         return 0;
6256 }
6257
6258 /**
6259  * Get age param from counter index.
6260  *
6261  * @param[in] dev
6262  *   Pointer to the Ethernet device structure.
6263  * @param[in] counter
6264  *   Index to the counter handler.
6265  *
6266  * @return
6267  *   The aging parameter specified for the counter index.
6268  */
6269 static struct mlx5_age_param*
6270 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
6271                                 uint32_t counter)
6272 {
6273         struct mlx5_flow_counter *cnt;
6274         struct mlx5_flow_counter_pool *pool = NULL;
6275
6276         flow_dv_counter_get_by_idx(dev, counter, &pool);
6277         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
6278         cnt = MLX5_POOL_GET_CNT(pool, counter);
6279         return MLX5_CNT_TO_AGE(cnt);
6280 }
6281
6282 /**
6283  * Remove a flow counter from aged counter list.
6284  *
6285  * @param[in] dev
6286  *   Pointer to the Ethernet device structure.
6287  * @param[in] counter
6288  *   Index to the counter handler.
6289  * @param[in] cnt
6290  *   Pointer to the counter handler.
6291  */
6292 static void
6293 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
6294                                 uint32_t counter, struct mlx5_flow_counter *cnt)
6295 {
6296         struct mlx5_age_info *age_info;
6297         struct mlx5_age_param *age_param;
6298         struct mlx5_priv *priv = dev->data->dev_private;
6299         uint16_t expected = AGE_CANDIDATE;
6300
6301         age_info = GET_PORT_AGE_INFO(priv);
6302         age_param = flow_dv_counter_idx_get_age(dev, counter);
6303         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
6304                                          AGE_FREE, false, __ATOMIC_RELAXED,
6305                                          __ATOMIC_RELAXED)) {
6306                 /**
6307                  * We need the lock even it is age timeout,
6308                  * since counter may still in process.
6309                  */
6310                 rte_spinlock_lock(&age_info->aged_sl);
6311                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
6312                 rte_spinlock_unlock(&age_info->aged_sl);
6313                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
6314         }
6315 }
6316
6317 /**
6318  * Release a flow counter.
6319  *
6320  * @param[in] dev
6321  *   Pointer to the Ethernet device structure.
6322  * @param[in] counter
6323  *   Index to the counter handler.
6324  */
6325 static void
6326 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter)
6327 {
6328         struct mlx5_priv *priv = dev->data->dev_private;
6329         struct mlx5_flow_counter_pool *pool = NULL;
6330         struct mlx5_flow_counter *cnt;
6331         enum mlx5_counter_type cnt_type;
6332
6333         if (!counter)
6334                 return;
6335         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6336         MLX5_ASSERT(pool);
6337         if (pool->is_aged) {
6338                 flow_dv_counter_remove_from_age(dev, counter, cnt);
6339         } else {
6340                 /*
6341                  * If the counter action is shared by indirect action API,
6342                  * the atomic function reduces its references counter.
6343                  * If after the reduction the action is still referenced, the
6344                  * function returns here and does not release it.
6345                  * When the counter action is not shared by
6346                  * indirect action API, shared info is 1 before the reduction,
6347                  * so this condition is failed and function doesn't return here.
6348                  */
6349                 if (__atomic_sub_fetch(&cnt->shared_info.refcnt, 1,
6350                                        __ATOMIC_RELAXED))
6351                         return;
6352         }
6353         cnt->pool = pool;
6354         /*
6355          * Put the counter back to list to be updated in none fallback mode.
6356          * Currently, we are using two list alternately, while one is in query,
6357          * add the freed counter to the other list based on the pool query_gen
6358          * value. After query finishes, add counter the list to the global
6359          * container counter list. The list changes while query starts. In
6360          * this case, lock will not be needed as query callback and release
6361          * function both operate with the different list.
6362          */
6363         if (!priv->sh->cmng.counter_fallback) {
6364                 rte_spinlock_lock(&pool->csl);
6365                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
6366                 rte_spinlock_unlock(&pool->csl);
6367         } else {
6368                 cnt->dcs_when_free = cnt->dcs_when_active;
6369                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6370                                            MLX5_COUNTER_TYPE_ORIGIN;
6371                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
6372                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
6373                                   cnt, next);
6374                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
6375         }
6376 }
6377
6378 /**
6379  * Resize a meter id container.
6380  *
6381  * @param[in] dev
6382  *   Pointer to the Ethernet device structure.
6383  *
6384  * @return
6385  *   0 on success, otherwise negative errno value and rte_errno is set.
6386  */
6387 static int
6388 flow_dv_mtr_container_resize(struct rte_eth_dev *dev)
6389 {
6390         struct mlx5_priv *priv = dev->data->dev_private;
6391         struct mlx5_aso_mtr_pools_mng *pools_mng =
6392                                 &priv->sh->mtrmng->pools_mng;
6393         void *old_pools = pools_mng->pools;
6394         uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE;
6395         uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize;
6396         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
6397
6398         if (!pools) {
6399                 rte_errno = ENOMEM;
6400                 return -ENOMEM;
6401         }
6402         if (!pools_mng->n)
6403                 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) {
6404                         mlx5_free(pools);
6405                         return -ENOMEM;
6406                 }
6407         if (old_pools)
6408                 memcpy(pools, old_pools, pools_mng->n *
6409                                        sizeof(struct mlx5_aso_mtr_pool *));
6410         pools_mng->n = resize;
6411         pools_mng->pools = pools;
6412         if (old_pools)
6413                 mlx5_free(old_pools);
6414         return 0;
6415 }
6416
6417 /**
6418  * Prepare a new meter and/or a new meter pool.
6419  *
6420  * @param[in] dev
6421  *   Pointer to the Ethernet device structure.
6422  * @param[out] mtr_free
6423  *   Where to put the pointer of a new meter.g.
6424  *
6425  * @return
6426  *   The meter pool pointer and @mtr_free is set on success,
6427  *   NULL otherwise and rte_errno is set.
6428  */
6429 static struct mlx5_aso_mtr_pool *
6430 flow_dv_mtr_pool_create(struct rte_eth_dev *dev, struct mlx5_aso_mtr **mtr_free)
6431 {
6432         struct mlx5_priv *priv = dev->data->dev_private;
6433         struct mlx5_aso_mtr_pools_mng *pools_mng = &priv->sh->mtrmng->pools_mng;
6434         struct mlx5_aso_mtr_pool *pool = NULL;
6435         struct mlx5_devx_obj *dcs = NULL;
6436         uint32_t i;
6437         uint32_t log_obj_size;
6438
6439         log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
6440         dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->cdev->ctx,
6441                                                       priv->sh->cdev->pdn,
6442                                                       log_obj_size);
6443         if (!dcs) {
6444                 rte_errno = ENODATA;
6445                 return NULL;
6446         }
6447         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
6448         if (!pool) {
6449                 rte_errno = ENOMEM;
6450                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6451                 return NULL;
6452         }
6453         pool->devx_obj = dcs;
6454         rte_rwlock_write_lock(&pools_mng->resize_mtrwl);
6455         pool->index = pools_mng->n_valid;
6456         if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) {
6457                 mlx5_free(pool);
6458                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6459                 rte_rwlock_write_unlock(&pools_mng->resize_mtrwl);
6460                 return NULL;
6461         }
6462         pools_mng->pools[pool->index] = pool;
6463         pools_mng->n_valid++;
6464         rte_rwlock_write_unlock(&pools_mng->resize_mtrwl);
6465         for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) {
6466                 pool->mtrs[i].offset = i;
6467                 LIST_INSERT_HEAD(&pools_mng->meters, &pool->mtrs[i], next);
6468         }
6469         pool->mtrs[0].offset = 0;
6470         *mtr_free = &pool->mtrs[0];
6471         return pool;
6472 }
6473
6474 /**
6475  * Release a flow meter into pool.
6476  *
6477  * @param[in] dev
6478  *   Pointer to the Ethernet device structure.
6479  * @param[in] mtr_idx
6480  *   Index to aso flow meter.
6481  */
6482 static void
6483 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx)
6484 {
6485         struct mlx5_priv *priv = dev->data->dev_private;
6486         struct mlx5_aso_mtr_pools_mng *pools_mng =
6487                                 &priv->sh->mtrmng->pools_mng;
6488         struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
6489
6490         MLX5_ASSERT(aso_mtr);
6491         rte_spinlock_lock(&pools_mng->mtrsl);
6492         memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info));
6493         aso_mtr->state = ASO_METER_FREE;
6494         LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next);
6495         rte_spinlock_unlock(&pools_mng->mtrsl);
6496 }
6497
6498 /**
6499  * Allocate a aso flow meter.
6500  *
6501  * @param[in] dev
6502  *   Pointer to the Ethernet device structure.
6503  *
6504  * @return
6505  *   Index to aso flow meter on success, 0 otherwise and rte_errno is set.
6506  */
6507 static uint32_t
6508 flow_dv_mtr_alloc(struct rte_eth_dev *dev)
6509 {
6510         struct mlx5_priv *priv = dev->data->dev_private;
6511         struct mlx5_aso_mtr *mtr_free = NULL;
6512         struct mlx5_aso_mtr_pools_mng *pools_mng =
6513                                 &priv->sh->mtrmng->pools_mng;
6514         struct mlx5_aso_mtr_pool *pool;
6515         uint32_t mtr_idx = 0;
6516
6517         if (!priv->sh->cdev->config.devx) {
6518                 rte_errno = ENOTSUP;
6519                 return 0;
6520         }
6521         /* Allocate the flow meter memory. */
6522         /* Get free meters from management. */
6523         rte_spinlock_lock(&pools_mng->mtrsl);
6524         mtr_free = LIST_FIRST(&pools_mng->meters);
6525         if (mtr_free)
6526                 LIST_REMOVE(mtr_free, next);
6527         if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) {
6528                 rte_spinlock_unlock(&pools_mng->mtrsl);
6529                 return 0;
6530         }
6531         mtr_free->state = ASO_METER_WAIT;
6532         rte_spinlock_unlock(&pools_mng->mtrsl);
6533         pool = container_of(mtr_free,
6534                         struct mlx5_aso_mtr_pool,
6535                         mtrs[mtr_free->offset]);
6536         mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
6537         if (!mtr_free->fm.meter_action) {
6538 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
6539                 struct rte_flow_error error;
6540                 uint8_t reg_id;
6541
6542                 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
6543                 mtr_free->fm.meter_action =
6544                         mlx5_glue->dv_create_flow_action_aso
6545                                                 (priv->sh->rx_domain,
6546                                                  pool->devx_obj->obj,
6547                                                  mtr_free->offset,
6548                                                  (1 << MLX5_FLOW_COLOR_GREEN),
6549                                                  reg_id - REG_C_0);
6550 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
6551                 if (!mtr_free->fm.meter_action) {
6552                         flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
6553                         return 0;
6554                 }
6555         }
6556         return mtr_idx;
6557 }
6558
6559 /**
6560  * Verify the @p attributes will be correctly understood by the NIC and store
6561  * them in the @p flow if everything is correct.
6562  *
6563  * @param[in] dev
6564  *   Pointer to dev struct.
6565  * @param[in] attributes
6566  *   Pointer to flow attributes
6567  * @param[in] external
6568  *   This flow rule is created by request external to PMD.
6569  * @param[out] error
6570  *   Pointer to error structure.
6571  *
6572  * @return
6573  *   - 0 on success and non root table.
6574  *   - 1 on success and root table.
6575  *   - a negative errno value otherwise and rte_errno is set.
6576  */
6577 static int
6578 flow_dv_validate_attributes(struct rte_eth_dev *dev,
6579                             const struct mlx5_flow_tunnel *tunnel,
6580                             const struct rte_flow_attr *attributes,
6581                             const struct flow_grp_info *grp_info,
6582                             struct rte_flow_error *error)
6583 {
6584         struct mlx5_priv *priv = dev->data->dev_private;
6585         uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes);
6586         int ret = 0;
6587
6588 #ifndef HAVE_MLX5DV_DR
6589         RTE_SET_USED(tunnel);
6590         RTE_SET_USED(grp_info);
6591         if (attributes->group)
6592                 return rte_flow_error_set(error, ENOTSUP,
6593                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6594                                           NULL,
6595                                           "groups are not supported");
6596 #else
6597         uint32_t table = 0;
6598
6599         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
6600                                        grp_info, error);
6601         if (ret)
6602                 return ret;
6603         if (!table)
6604                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
6605 #endif
6606         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
6607             attributes->priority > lowest_priority)
6608                 return rte_flow_error_set(error, ENOTSUP,
6609                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
6610                                           NULL,
6611                                           "priority out of range");
6612         if (attributes->transfer) {
6613                 if (!priv->sh->config.dv_esw_en)
6614                         return rte_flow_error_set
6615                                 (error, ENOTSUP,
6616                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6617                                  "E-Switch dr is not supported");
6618                 if (attributes->egress)
6619                         return rte_flow_error_set
6620                                 (error, ENOTSUP,
6621                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
6622                                  "egress is not supported");
6623         }
6624         if (!(attributes->egress ^ attributes->ingress))
6625                 return rte_flow_error_set(error, ENOTSUP,
6626                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
6627                                           "must specify exactly one of "
6628                                           "ingress or egress");
6629         return ret;
6630 }
6631
6632 static int
6633 validate_integrity_bits(const struct rte_flow_item_integrity *mask,
6634                         int64_t pattern_flags, uint64_t l3_flags,
6635                         uint64_t l4_flags, uint64_t ip4_flag,
6636                         struct rte_flow_error *error)
6637 {
6638         if (mask->l3_ok && !(pattern_flags & l3_flags))
6639                 return rte_flow_error_set(error, EINVAL,
6640                                           RTE_FLOW_ERROR_TYPE_ITEM,
6641                                           NULL, "missing L3 protocol");
6642
6643         if (mask->ipv4_csum_ok && !(pattern_flags & ip4_flag))
6644                 return rte_flow_error_set(error, EINVAL,
6645                                           RTE_FLOW_ERROR_TYPE_ITEM,
6646                                           NULL, "missing IPv4 protocol");
6647
6648         if ((mask->l4_ok || mask->l4_csum_ok) && !(pattern_flags & l4_flags))
6649                 return rte_flow_error_set(error, EINVAL,
6650                                           RTE_FLOW_ERROR_TYPE_ITEM,
6651                                           NULL, "missing L4 protocol");
6652
6653         return 0;
6654 }
6655
6656 static int
6657 flow_dv_validate_item_integrity_post(const struct
6658                                      rte_flow_item *integrity_items[2],
6659                                      int64_t pattern_flags,
6660                                      struct rte_flow_error *error)
6661 {
6662         const struct rte_flow_item_integrity *mask;
6663         int ret;
6664
6665         if (pattern_flags & MLX5_FLOW_ITEM_OUTER_INTEGRITY) {
6666                 mask = (typeof(mask))integrity_items[0]->mask;
6667                 ret = validate_integrity_bits(mask, pattern_flags,
6668                                               MLX5_FLOW_LAYER_OUTER_L3,
6669                                               MLX5_FLOW_LAYER_OUTER_L4,
6670                                               MLX5_FLOW_LAYER_OUTER_L3_IPV4,
6671                                               error);
6672                 if (ret)
6673                         return ret;
6674         }
6675         if (pattern_flags & MLX5_FLOW_ITEM_INNER_INTEGRITY) {
6676                 mask = (typeof(mask))integrity_items[1]->mask;
6677                 ret = validate_integrity_bits(mask, pattern_flags,
6678                                               MLX5_FLOW_LAYER_INNER_L3,
6679                                               MLX5_FLOW_LAYER_INNER_L4,
6680                                               MLX5_FLOW_LAYER_INNER_L3_IPV4,
6681                                               error);
6682                 if (ret)
6683                         return ret;
6684         }
6685         return 0;
6686 }
6687
6688 static int
6689 flow_dv_validate_item_integrity(struct rte_eth_dev *dev,
6690                                 const struct rte_flow_item *integrity_item,
6691                                 uint64_t pattern_flags, uint64_t *last_item,
6692                                 const struct rte_flow_item *integrity_items[2],
6693                                 struct rte_flow_error *error)
6694 {
6695         struct mlx5_priv *priv = dev->data->dev_private;
6696         const struct rte_flow_item_integrity *mask = (typeof(mask))
6697                                                      integrity_item->mask;
6698         const struct rte_flow_item_integrity *spec = (typeof(spec))
6699                                                      integrity_item->spec;
6700
6701         if (!priv->sh->cdev->config.hca_attr.pkt_integrity_match)
6702                 return rte_flow_error_set(error, ENOTSUP,
6703                                           RTE_FLOW_ERROR_TYPE_ITEM,
6704                                           integrity_item,
6705                                           "packet integrity integrity_item not supported");
6706         if (!spec)
6707                 return rte_flow_error_set(error, ENOTSUP,
6708                                           RTE_FLOW_ERROR_TYPE_ITEM,
6709                                           integrity_item,
6710                                           "no spec for integrity item");
6711         if (!mask)
6712                 mask = &rte_flow_item_integrity_mask;
6713         if (!mlx5_validate_integrity_item(mask))
6714                 return rte_flow_error_set(error, ENOTSUP,
6715                                           RTE_FLOW_ERROR_TYPE_ITEM,
6716                                           integrity_item,
6717                                           "unsupported integrity filter");
6718         if (spec->level > 1) {
6719                 if (pattern_flags & MLX5_FLOW_ITEM_INNER_INTEGRITY)
6720                         return rte_flow_error_set
6721                                 (error, ENOTSUP,
6722                                  RTE_FLOW_ERROR_TYPE_ITEM,
6723                                  NULL, "multiple inner integrity items not supported");
6724                 integrity_items[1] = integrity_item;
6725                 *last_item |= MLX5_FLOW_ITEM_INNER_INTEGRITY;
6726         } else {
6727                 if (pattern_flags & MLX5_FLOW_ITEM_OUTER_INTEGRITY)
6728                         return rte_flow_error_set
6729                                 (error, ENOTSUP,
6730                                  RTE_FLOW_ERROR_TYPE_ITEM,
6731                                  NULL, "multiple outer integrity items not supported");
6732                 integrity_items[0] = integrity_item;
6733                 *last_item |= MLX5_FLOW_ITEM_OUTER_INTEGRITY;
6734         }
6735         return 0;
6736 }
6737
6738 static int
6739 flow_dv_validate_item_flex(struct rte_eth_dev *dev,
6740                            const struct rte_flow_item *item,
6741                            uint64_t item_flags,
6742                            uint64_t *last_item,
6743                            bool is_inner,
6744                            struct rte_flow_error *error)
6745 {
6746         const struct rte_flow_item_flex *flow_spec = item->spec;
6747         const struct rte_flow_item_flex *flow_mask = item->mask;
6748         struct mlx5_flex_item *flex;
6749
6750         if (!flow_spec)
6751                 return rte_flow_error_set(error, EINVAL,
6752                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
6753                                           "flex flow item spec cannot be NULL");
6754         if (!flow_mask)
6755                 return rte_flow_error_set(error, EINVAL,
6756                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
6757                                           "flex flow item mask cannot be NULL");
6758         if (item->last)
6759                 return rte_flow_error_set(error, ENOTSUP,
6760                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
6761                                           "flex flow item last not supported");
6762         if (mlx5_flex_acquire_index(dev, flow_spec->handle, false) < 0)
6763                 return rte_flow_error_set(error, EINVAL,
6764                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
6765                                           "invalid flex flow item handle");
6766         flex = (struct mlx5_flex_item *)flow_spec->handle;
6767         switch (flex->tunnel_mode) {
6768         case FLEX_TUNNEL_MODE_SINGLE:
6769                 if (item_flags &
6770                     (MLX5_FLOW_ITEM_OUTER_FLEX | MLX5_FLOW_ITEM_INNER_FLEX))
6771                         rte_flow_error_set(error, EINVAL,
6772                                            RTE_FLOW_ERROR_TYPE_ITEM,
6773                                            NULL, "multiple flex items not supported");
6774                 break;
6775         case FLEX_TUNNEL_MODE_OUTER:
6776                 if (is_inner)
6777                         rte_flow_error_set(error, EINVAL,
6778                                            RTE_FLOW_ERROR_TYPE_ITEM,
6779                                            NULL, "inner flex item was not configured");
6780                 if (item_flags & MLX5_FLOW_ITEM_OUTER_FLEX)
6781                         rte_flow_error_set(error, ENOTSUP,
6782                                            RTE_FLOW_ERROR_TYPE_ITEM,
6783                                            NULL, "multiple flex items not supported");
6784                 break;
6785         case FLEX_TUNNEL_MODE_INNER:
6786                 if (!is_inner)
6787                         rte_flow_error_set(error, EINVAL,
6788                                            RTE_FLOW_ERROR_TYPE_ITEM,
6789                                            NULL, "outer flex item was not configured");
6790                 if (item_flags & MLX5_FLOW_ITEM_INNER_FLEX)
6791                         rte_flow_error_set(error, EINVAL,
6792                                            RTE_FLOW_ERROR_TYPE_ITEM,
6793                                            NULL, "multiple flex items not supported");
6794                 break;
6795         case FLEX_TUNNEL_MODE_MULTI:
6796                 if ((is_inner && (item_flags & MLX5_FLOW_ITEM_INNER_FLEX)) ||
6797                     (!is_inner && (item_flags & MLX5_FLOW_ITEM_OUTER_FLEX))) {
6798                         rte_flow_error_set(error, EINVAL,
6799                                            RTE_FLOW_ERROR_TYPE_ITEM,
6800                                            NULL, "multiple flex items not supported");
6801                 }
6802                 break;
6803         case FLEX_TUNNEL_MODE_TUNNEL:
6804                 if (is_inner || (item_flags & MLX5_FLOW_ITEM_FLEX_TUNNEL))
6805                         rte_flow_error_set(error, EINVAL,
6806                                            RTE_FLOW_ERROR_TYPE_ITEM,
6807                                            NULL, "multiple flex tunnel items not supported");
6808                 break;
6809         default:
6810                 rte_flow_error_set(error, EINVAL,
6811                                    RTE_FLOW_ERROR_TYPE_ITEM,
6812                                    NULL, "invalid flex item configuration");
6813         }
6814         *last_item = flex->tunnel_mode == FLEX_TUNNEL_MODE_TUNNEL ?
6815                      MLX5_FLOW_ITEM_FLEX_TUNNEL : is_inner ?
6816                      MLX5_FLOW_ITEM_INNER_FLEX : MLX5_FLOW_ITEM_OUTER_FLEX;
6817         return 0;
6818 }
6819
6820 /**
6821  * Internal validation function. For validating both actions and items.
6822  *
6823  * @param[in] dev
6824  *   Pointer to the rte_eth_dev structure.
6825  * @param[in] attr
6826  *   Pointer to the flow attributes.
6827  * @param[in] items
6828  *   Pointer to the list of items.
6829  * @param[in] actions
6830  *   Pointer to the list of actions.
6831  * @param[in] external
6832  *   This flow rule is created by request external to PMD.
6833  * @param[in] hairpin
6834  *   Number of hairpin TX actions, 0 means classic flow.
6835  * @param[out] error
6836  *   Pointer to the error structure.
6837  *
6838  * @return
6839  *   0 on success, a negative errno value otherwise and rte_errno is set.
6840  */
6841 static int
6842 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
6843                  const struct rte_flow_item items[],
6844                  const struct rte_flow_action actions[],
6845                  bool external, int hairpin, struct rte_flow_error *error)
6846 {
6847         int ret;
6848         uint64_t aso_mask, action_flags = 0;
6849         uint64_t item_flags = 0;
6850         uint64_t last_item = 0;
6851         uint8_t next_protocol = 0xff;
6852         uint16_t ether_type = 0;
6853         int actions_n = 0;
6854         uint8_t item_ipv6_proto = 0;
6855         int fdb_mirror_limit = 0;
6856         int modify_after_mirror = 0;
6857         const struct rte_flow_item *geneve_item = NULL;
6858         const struct rte_flow_item *gre_item = NULL;
6859         const struct rte_flow_item *gtp_item = NULL;
6860         const struct rte_flow_action_raw_decap *decap;
6861         const struct rte_flow_action_raw_encap *encap;
6862         const struct rte_flow_action_rss *rss = NULL;
6863         const struct rte_flow_action_rss *sample_rss = NULL;
6864         const struct rte_flow_action_count *sample_count = NULL;
6865         const struct rte_flow_item_tcp nic_tcp_mask = {
6866                 .hdr = {
6867                         .tcp_flags = 0xFF,
6868                         .src_port = RTE_BE16(UINT16_MAX),
6869                         .dst_port = RTE_BE16(UINT16_MAX),
6870                 }
6871         };
6872         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
6873                 .hdr = {
6874                         .src_addr =
6875                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6876                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6877                         .dst_addr =
6878                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6879                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6880                         .vtc_flow = RTE_BE32(0xffffffff),
6881                         .proto = 0xff,
6882                         .hop_limits = 0xff,
6883                 },
6884                 .has_frag_ext = 1,
6885         };
6886         const struct rte_flow_item_ecpri nic_ecpri_mask = {
6887                 .hdr = {
6888                         .common = {
6889                                 .u32 =
6890                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
6891                                         .type = 0xFF,
6892                                         }).u32),
6893                         },
6894                         .dummy[0] = 0xffffffff,
6895                 },
6896         };
6897         struct mlx5_priv *priv = dev->data->dev_private;
6898         struct mlx5_sh_config *dev_conf = &priv->sh->config;
6899         uint16_t queue_index = 0xFFFF;
6900         const struct rte_flow_item_vlan *vlan_m = NULL;
6901         uint32_t rw_act_num = 0;
6902         uint64_t is_root;
6903         const struct mlx5_flow_tunnel *tunnel;
6904         enum mlx5_tof_rule_type tof_rule_type;
6905         struct flow_grp_info grp_info = {
6906                 .external = !!external,
6907                 .transfer = !!attr->transfer,
6908                 .fdb_def_rule = !!priv->fdb_def_rule,
6909                 .std_tbl_fix = true,
6910         };
6911         const struct rte_eth_hairpin_conf *conf;
6912         const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
6913         const struct rte_flow_item *port_id_item = NULL;
6914         bool def_policy = false;
6915         bool shared_count = false;
6916         uint16_t udp_dport = 0;
6917         uint32_t tag_id = 0;
6918         const struct rte_flow_action_age *non_shared_age = NULL;
6919         const struct rte_flow_action_count *count = NULL;
6920
6921         if (items == NULL)
6922                 return -1;
6923         tunnel = is_tunnel_offload_active(dev) ?
6924                  mlx5_get_tof(items, actions, &tof_rule_type) : NULL;
6925         if (tunnel) {
6926                 if (!dev_conf->dv_flow_en)
6927                         return rte_flow_error_set
6928                                 (error, ENOTSUP,
6929                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6930                                  NULL, "tunnel offload requires DV flow interface");
6931                 if (priv->representor)
6932                         return rte_flow_error_set
6933                                 (error, ENOTSUP,
6934                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6935                                  NULL, "decap not supported for VF representor");
6936                 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE)
6937                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6938                 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE)
6939                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
6940                                         MLX5_FLOW_ACTION_DECAP;
6941                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
6942                                         (dev, attr, tunnel, tof_rule_type);
6943         }
6944         ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error);
6945         if (ret < 0)
6946                 return ret;
6947         is_root = (uint64_t)ret;
6948         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6949                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6950                 int type = items->type;
6951
6952                 if (!mlx5_flow_os_item_supported(type))
6953                         return rte_flow_error_set(error, ENOTSUP,
6954                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6955                                                   NULL, "item not supported");
6956                 switch (type) {
6957                 case RTE_FLOW_ITEM_TYPE_VOID:
6958                         break;
6959                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6960                         ret = flow_dv_validate_item_port_id
6961                                         (dev, items, attr, item_flags, error);
6962                         if (ret < 0)
6963                                 return ret;
6964                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6965                         port_id_item = items;
6966                         break;
6967                 case RTE_FLOW_ITEM_TYPE_ETH:
6968                         ret = mlx5_flow_validate_item_eth(items, item_flags,
6969                                                           true, error);
6970                         if (ret < 0)
6971                                 return ret;
6972                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
6973                                              MLX5_FLOW_LAYER_OUTER_L2;
6974                         if (items->mask != NULL && items->spec != NULL) {
6975                                 ether_type =
6976                                         ((const struct rte_flow_item_eth *)
6977                                          items->spec)->type;
6978                                 ether_type &=
6979                                         ((const struct rte_flow_item_eth *)
6980                                          items->mask)->type;
6981                                 ether_type = rte_be_to_cpu_16(ether_type);
6982                         } else {
6983                                 ether_type = 0;
6984                         }
6985                         break;
6986                 case RTE_FLOW_ITEM_TYPE_VLAN:
6987                         ret = flow_dv_validate_item_vlan(items, item_flags,
6988                                                          dev, error);
6989                         if (ret < 0)
6990                                 return ret;
6991                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
6992                                              MLX5_FLOW_LAYER_OUTER_VLAN;
6993                         if (items->mask != NULL && items->spec != NULL) {
6994                                 ether_type =
6995                                         ((const struct rte_flow_item_vlan *)
6996                                          items->spec)->inner_type;
6997                                 ether_type &=
6998                                         ((const struct rte_flow_item_vlan *)
6999                                          items->mask)->inner_type;
7000                                 ether_type = rte_be_to_cpu_16(ether_type);
7001                         } else {
7002                                 ether_type = 0;
7003                         }
7004                         /* Store outer VLAN mask for of_push_vlan action. */
7005                         if (!tunnel)
7006                                 vlan_m = items->mask;
7007                         break;
7008                 case RTE_FLOW_ITEM_TYPE_IPV4:
7009                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7010                                                   &item_flags, &tunnel);
7011                         ret = flow_dv_validate_item_ipv4(dev, items, item_flags,
7012                                                          last_item, ether_type,
7013                                                          error);
7014                         if (ret < 0)
7015                                 return ret;
7016                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
7017                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
7018                         if (items->mask != NULL &&
7019                             ((const struct rte_flow_item_ipv4 *)
7020                              items->mask)->hdr.next_proto_id) {
7021                                 next_protocol =
7022                                         ((const struct rte_flow_item_ipv4 *)
7023                                          (items->spec))->hdr.next_proto_id;
7024                                 next_protocol &=
7025                                         ((const struct rte_flow_item_ipv4 *)
7026                                          (items->mask))->hdr.next_proto_id;
7027                         } else {
7028                                 /* Reset for inner layer. */
7029                                 next_protocol = 0xff;
7030                         }
7031                         break;
7032                 case RTE_FLOW_ITEM_TYPE_IPV6:
7033                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7034                                                   &item_flags, &tunnel);
7035                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
7036                                                            last_item,
7037                                                            ether_type,
7038                                                            &nic_ipv6_mask,
7039                                                            error);
7040                         if (ret < 0)
7041                                 return ret;
7042                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
7043                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
7044                         if (items->mask != NULL &&
7045                             ((const struct rte_flow_item_ipv6 *)
7046                              items->mask)->hdr.proto) {
7047                                 item_ipv6_proto =
7048                                         ((const struct rte_flow_item_ipv6 *)
7049                                          items->spec)->hdr.proto;
7050                                 next_protocol =
7051                                         ((const struct rte_flow_item_ipv6 *)
7052                                          items->spec)->hdr.proto;
7053                                 next_protocol &=
7054                                         ((const struct rte_flow_item_ipv6 *)
7055                                          items->mask)->hdr.proto;
7056                         } else {
7057                                 /* Reset for inner layer. */
7058                                 next_protocol = 0xff;
7059                         }
7060                         break;
7061                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
7062                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
7063                                                                   item_flags,
7064                                                                   error);
7065                         if (ret < 0)
7066                                 return ret;
7067                         last_item = tunnel ?
7068                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
7069                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
7070                         if (items->mask != NULL &&
7071                             ((const struct rte_flow_item_ipv6_frag_ext *)
7072                              items->mask)->hdr.next_header) {
7073                                 next_protocol =
7074                                 ((const struct rte_flow_item_ipv6_frag_ext *)
7075                                  items->spec)->hdr.next_header;
7076                                 next_protocol &=
7077                                 ((const struct rte_flow_item_ipv6_frag_ext *)
7078                                  items->mask)->hdr.next_header;
7079                         } else {
7080                                 /* Reset for inner layer. */
7081                                 next_protocol = 0xff;
7082                         }
7083                         break;
7084                 case RTE_FLOW_ITEM_TYPE_TCP:
7085                         ret = mlx5_flow_validate_item_tcp
7086                                                 (items, item_flags,
7087                                                  next_protocol,
7088                                                  &nic_tcp_mask,
7089                                                  error);
7090                         if (ret < 0)
7091                                 return ret;
7092                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
7093                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
7094                         break;
7095                 case RTE_FLOW_ITEM_TYPE_UDP:
7096                         ret = mlx5_flow_validate_item_udp(items, item_flags,
7097                                                           next_protocol,
7098                                                           error);
7099                         const struct rte_flow_item_udp *spec = items->spec;
7100                         const struct rte_flow_item_udp *mask = items->mask;
7101                         if (!mask)
7102                                 mask = &rte_flow_item_udp_mask;
7103                         if (spec != NULL)
7104                                 udp_dport = rte_be_to_cpu_16
7105                                                 (spec->hdr.dst_port &
7106                                                  mask->hdr.dst_port);
7107                         if (ret < 0)
7108                                 return ret;
7109                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
7110                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
7111                         break;
7112                 case RTE_FLOW_ITEM_TYPE_GRE:
7113                         ret = mlx5_flow_validate_item_gre(items, item_flags,
7114                                                           next_protocol, error);
7115                         if (ret < 0)
7116                                 return ret;
7117                         gre_item = items;
7118                         last_item = MLX5_FLOW_LAYER_GRE;
7119                         break;
7120                 case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
7121                         ret = mlx5_flow_validate_item_gre_option(dev, items, item_flags,
7122                                                           attr, gre_item, error);
7123                         if (ret < 0)
7124                                 return ret;
7125                         last_item = MLX5_FLOW_LAYER_GRE;
7126                         break;
7127                 case RTE_FLOW_ITEM_TYPE_NVGRE:
7128                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
7129                                                             next_protocol,
7130                                                             error);
7131                         if (ret < 0)
7132                                 return ret;
7133                         last_item = MLX5_FLOW_LAYER_NVGRE;
7134                         break;
7135                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
7136                         ret = mlx5_flow_validate_item_gre_key
7137                                 (items, item_flags, gre_item, error);
7138                         if (ret < 0)
7139                                 return ret;
7140                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
7141                         break;
7142                 case RTE_FLOW_ITEM_TYPE_VXLAN:
7143                         ret = mlx5_flow_validate_item_vxlan(dev, udp_dport,
7144                                                             items, item_flags,
7145                                                             attr, error);
7146                         if (ret < 0)
7147                                 return ret;
7148                         last_item = MLX5_FLOW_LAYER_VXLAN;
7149                         break;
7150                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
7151                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
7152                                                                 item_flags, dev,
7153                                                                 error);
7154                         if (ret < 0)
7155                                 return ret;
7156                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
7157                         break;
7158                 case RTE_FLOW_ITEM_TYPE_GENEVE:
7159                         ret = mlx5_flow_validate_item_geneve(items,
7160                                                              item_flags, dev,
7161                                                              error);
7162                         if (ret < 0)
7163                                 return ret;
7164                         geneve_item = items;
7165                         last_item = MLX5_FLOW_LAYER_GENEVE;
7166                         break;
7167                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
7168                         ret = mlx5_flow_validate_item_geneve_opt(items,
7169                                                                  last_item,
7170                                                                  geneve_item,
7171                                                                  dev,
7172                                                                  error);
7173                         if (ret < 0)
7174                                 return ret;
7175                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
7176                         break;
7177                 case RTE_FLOW_ITEM_TYPE_MPLS:
7178                         ret = mlx5_flow_validate_item_mpls(dev, items,
7179                                                            item_flags,
7180                                                            last_item, error);
7181                         if (ret < 0)
7182                                 return ret;
7183                         last_item = MLX5_FLOW_LAYER_MPLS;
7184                         break;
7185
7186                 case RTE_FLOW_ITEM_TYPE_MARK:
7187                         ret = flow_dv_validate_item_mark(dev, items, attr,
7188                                                          error);
7189                         if (ret < 0)
7190                                 return ret;
7191                         last_item = MLX5_FLOW_ITEM_MARK;
7192                         break;
7193                 case RTE_FLOW_ITEM_TYPE_META:
7194                         ret = flow_dv_validate_item_meta(dev, items, attr,
7195                                                          error);
7196                         if (ret < 0)
7197                                 return ret;
7198                         last_item = MLX5_FLOW_ITEM_METADATA;
7199                         break;
7200                 case RTE_FLOW_ITEM_TYPE_ICMP:
7201                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
7202                                                            next_protocol,
7203                                                            error);
7204                         if (ret < 0)
7205                                 return ret;
7206                         last_item = MLX5_FLOW_LAYER_ICMP;
7207                         break;
7208                 case RTE_FLOW_ITEM_TYPE_ICMP6:
7209                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
7210                                                             next_protocol,
7211                                                             error);
7212                         if (ret < 0)
7213                                 return ret;
7214                         item_ipv6_proto = IPPROTO_ICMPV6;
7215                         last_item = MLX5_FLOW_LAYER_ICMP6;
7216                         break;
7217                 case RTE_FLOW_ITEM_TYPE_TAG:
7218                         ret = flow_dv_validate_item_tag(dev, items,
7219                                                         attr, error);
7220                         if (ret < 0)
7221                                 return ret;
7222                         last_item = MLX5_FLOW_ITEM_TAG;
7223                         break;
7224                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
7225                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
7226                         break;
7227                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
7228                         break;
7229                 case RTE_FLOW_ITEM_TYPE_GTP:
7230                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
7231                                                         error);
7232                         if (ret < 0)
7233                                 return ret;
7234                         gtp_item = items;
7235                         last_item = MLX5_FLOW_LAYER_GTP;
7236                         break;
7237                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
7238                         ret = flow_dv_validate_item_gtp_psc(items, last_item,
7239                                                             gtp_item, attr,
7240                                                             error);
7241                         if (ret < 0)
7242                                 return ret;
7243                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
7244                         break;
7245                 case RTE_FLOW_ITEM_TYPE_ECPRI:
7246                         /* Capacity will be checked in the translate stage. */
7247                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
7248                                                             last_item,
7249                                                             ether_type,
7250                                                             &nic_ecpri_mask,
7251                                                             error);
7252                         if (ret < 0)
7253                                 return ret;
7254                         last_item = MLX5_FLOW_LAYER_ECPRI;
7255                         break;
7256                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
7257                         ret = flow_dv_validate_item_integrity(dev, items,
7258                                                               item_flags,
7259                                                               &last_item,
7260                                                               integrity_items,
7261                                                               error);
7262                         if (ret < 0)
7263                                 return ret;
7264                         break;
7265                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
7266                         ret = flow_dv_validate_item_aso_ct(dev, items,
7267                                                            &item_flags, error);
7268                         if (ret < 0)
7269                                 return ret;
7270                         break;
7271                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
7272                         /* tunnel offload item was processed before
7273                          * list it here as a supported type
7274                          */
7275                         break;
7276                 case RTE_FLOW_ITEM_TYPE_FLEX:
7277                         ret = flow_dv_validate_item_flex(dev, items, item_flags,
7278                                                          &last_item,
7279                                                          tunnel != 0, error);
7280                         if (ret < 0)
7281                                 return ret;
7282                         break;
7283                 default:
7284                         return rte_flow_error_set(error, ENOTSUP,
7285                                                   RTE_FLOW_ERROR_TYPE_ITEM,
7286                                                   NULL, "item not supported");
7287                 }
7288                 item_flags |= last_item;
7289         }
7290         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY) {
7291                 ret = flow_dv_validate_item_integrity_post(integrity_items,
7292                                                            item_flags, error);
7293                 if (ret)
7294                         return ret;
7295         }
7296         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7297                 int type = actions->type;
7298
7299                 if (!mlx5_flow_os_action_supported(type))
7300                         return rte_flow_error_set(error, ENOTSUP,
7301                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7302                                                   actions,
7303                                                   "action not supported");
7304                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
7305                         return rte_flow_error_set(error, ENOTSUP,
7306                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7307                                                   actions, "too many actions");
7308                 if (action_flags &
7309                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
7310                         return rte_flow_error_set(error, ENOTSUP,
7311                                 RTE_FLOW_ERROR_TYPE_ACTION,
7312                                 NULL, "meter action with policy "
7313                                 "must be the last action");
7314                 switch (type) {
7315                 case RTE_FLOW_ACTION_TYPE_VOID:
7316                         break;
7317                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7318                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
7319                         ret = flow_dv_validate_action_port_id(dev,
7320                                                               action_flags,
7321                                                               actions,
7322                                                               attr,
7323                                                               error);
7324                         if (ret)
7325                                 return ret;
7326                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7327                         ++actions_n;
7328                         break;
7329                 case RTE_FLOW_ACTION_TYPE_FLAG:
7330                         ret = flow_dv_validate_action_flag(dev, action_flags,
7331                                                            attr, error);
7332                         if (ret < 0)
7333                                 return ret;
7334                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7335                                 /* Count all modify-header actions as one. */
7336                                 if (!(action_flags &
7337                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7338                                         ++actions_n;
7339                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
7340                                                 MLX5_FLOW_ACTION_MARK_EXT;
7341                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7342                                         modify_after_mirror = 1;
7343
7344                         } else {
7345                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
7346                                 ++actions_n;
7347                         }
7348                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7349                         break;
7350                 case RTE_FLOW_ACTION_TYPE_MARK:
7351                         ret = flow_dv_validate_action_mark(dev, actions,
7352                                                            action_flags,
7353                                                            attr, error);
7354                         if (ret < 0)
7355                                 return ret;
7356                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7357                                 /* Count all modify-header actions as one. */
7358                                 if (!(action_flags &
7359                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7360                                         ++actions_n;
7361                                 action_flags |= MLX5_FLOW_ACTION_MARK |
7362                                                 MLX5_FLOW_ACTION_MARK_EXT;
7363                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7364                                         modify_after_mirror = 1;
7365                         } else {
7366                                 action_flags |= MLX5_FLOW_ACTION_MARK;
7367                                 ++actions_n;
7368                         }
7369                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7370                         break;
7371                 case RTE_FLOW_ACTION_TYPE_SET_META:
7372                         ret = flow_dv_validate_action_set_meta(dev, actions,
7373                                                                action_flags,
7374                                                                attr, error);
7375                         if (ret < 0)
7376                                 return ret;
7377                         /* Count all modify-header actions as one action. */
7378                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7379                                 ++actions_n;
7380                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7381                                 modify_after_mirror = 1;
7382                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7383                         rw_act_num += MLX5_ACT_NUM_SET_META;
7384                         break;
7385                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7386                         ret = flow_dv_validate_action_set_tag(dev, actions,
7387                                                               action_flags,
7388                                                               attr, error);
7389                         if (ret < 0)
7390                                 return ret;
7391                         /* Count all modify-header actions as one action. */
7392                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7393                                 ++actions_n;
7394                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7395                                 modify_after_mirror = 1;
7396                         tag_id = ((const struct rte_flow_action_set_tag *)
7397                                   actions->conf)->index;
7398                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7399                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7400                         break;
7401                 case RTE_FLOW_ACTION_TYPE_DROP:
7402                         ret = mlx5_flow_validate_action_drop(action_flags,
7403                                                              attr, error);
7404                         if (ret < 0)
7405                                 return ret;
7406                         action_flags |= MLX5_FLOW_ACTION_DROP;
7407                         ++actions_n;
7408                         break;
7409                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7410                         ret = mlx5_flow_validate_action_queue(actions,
7411                                                               action_flags, dev,
7412                                                               attr, error);
7413                         if (ret < 0)
7414                                 return ret;
7415                         queue_index = ((const struct rte_flow_action_queue *)
7416                                                         (actions->conf))->index;
7417                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7418                         ++actions_n;
7419                         break;
7420                 case RTE_FLOW_ACTION_TYPE_RSS:
7421                         rss = actions->conf;
7422                         ret = mlx5_flow_validate_action_rss(actions,
7423                                                             action_flags, dev,
7424                                                             attr, item_flags,
7425                                                             error);
7426                         if (ret < 0)
7427                                 return ret;
7428                         if (rss && sample_rss &&
7429                             (sample_rss->level != rss->level ||
7430                             sample_rss->types != rss->types))
7431                                 return rte_flow_error_set(error, ENOTSUP,
7432                                         RTE_FLOW_ERROR_TYPE_ACTION,
7433                                         NULL,
7434                                         "Can't use the different RSS types "
7435                                         "or level in the same flow");
7436                         if (rss != NULL && rss->queue_num)
7437                                 queue_index = rss->queue[0];
7438                         action_flags |= MLX5_FLOW_ACTION_RSS;
7439                         ++actions_n;
7440                         break;
7441                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
7442                         ret =
7443                         mlx5_flow_validate_action_default_miss(action_flags,
7444                                         attr, error);
7445                         if (ret < 0)
7446                                 return ret;
7447                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
7448                         ++actions_n;
7449                         break;
7450                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
7451                         shared_count = true;
7452                         /* fall-through. */
7453                 case RTE_FLOW_ACTION_TYPE_COUNT:
7454                         ret = flow_dv_validate_action_count(dev, shared_count,
7455                                                             action_flags,
7456                                                             attr, error);
7457                         if (ret < 0)
7458                                 return ret;
7459                         count = actions->conf;
7460                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7461                         ++actions_n;
7462                         break;
7463                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7464                         if (flow_dv_validate_action_pop_vlan(dev,
7465                                                              action_flags,
7466                                                              actions,
7467                                                              item_flags, attr,
7468                                                              error))
7469                                 return -rte_errno;
7470                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7471                                 modify_after_mirror = 1;
7472                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7473                         ++actions_n;
7474                         break;
7475                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
7476                         ret = flow_dv_validate_action_push_vlan(dev,
7477                                                                 action_flags,
7478                                                                 vlan_m,
7479                                                                 actions, attr,
7480                                                                 error);
7481                         if (ret < 0)
7482                                 return ret;
7483                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7484                                 modify_after_mirror = 1;
7485                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
7486                         ++actions_n;
7487                         break;
7488                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
7489                         ret = flow_dv_validate_action_set_vlan_pcp
7490                                                 (action_flags, actions, error);
7491                         if (ret < 0)
7492                                 return ret;
7493                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7494                                 modify_after_mirror = 1;
7495                         /* Count PCP with push_vlan command. */
7496                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
7497                         break;
7498                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
7499                         ret = flow_dv_validate_action_set_vlan_vid
7500                                                 (item_flags, action_flags,
7501                                                  actions, error);
7502                         if (ret < 0)
7503                                 return ret;
7504                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7505                                 modify_after_mirror = 1;
7506                         /* Count VID with push_vlan command. */
7507                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
7508                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
7509                         break;
7510                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
7511                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
7512                         ret = flow_dv_validate_action_l2_encap(dev,
7513                                                                action_flags,
7514                                                                actions, attr,
7515                                                                error);
7516                         if (ret < 0)
7517                                 return ret;
7518                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
7519                         ++actions_n;
7520                         break;
7521                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
7522                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
7523                         ret = flow_dv_validate_action_decap(dev, action_flags,
7524                                                             actions, item_flags,
7525                                                             attr, error);
7526                         if (ret < 0)
7527                                 return ret;
7528                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7529                                 modify_after_mirror = 1;
7530                         action_flags |= MLX5_FLOW_ACTION_DECAP;
7531                         ++actions_n;
7532                         break;
7533                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
7534                         ret = flow_dv_validate_action_raw_encap_decap
7535                                 (dev, NULL, actions->conf, attr, &action_flags,
7536                                  &actions_n, actions, item_flags, error);
7537                         if (ret < 0)
7538                                 return ret;
7539                         break;
7540                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
7541                         decap = actions->conf;
7542                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
7543                                 ;
7544                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
7545                                 encap = NULL;
7546                                 actions--;
7547                         } else {
7548                                 encap = actions->conf;
7549                         }
7550                         ret = flow_dv_validate_action_raw_encap_decap
7551                                            (dev,
7552                                             decap ? decap : &empty_decap, encap,
7553                                             attr, &action_flags, &actions_n,
7554                                             actions, item_flags, error);
7555                         if (ret < 0)
7556                                 return ret;
7557                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7558                             (action_flags & MLX5_FLOW_ACTION_DECAP))
7559                                 modify_after_mirror = 1;
7560                         break;
7561                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
7562                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
7563                         ret = flow_dv_validate_action_modify_mac(action_flags,
7564                                                                  actions,
7565                                                                  item_flags,
7566                                                                  error);
7567                         if (ret < 0)
7568                                 return ret;
7569                         /* Count all modify-header actions as one action. */
7570                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7571                                 ++actions_n;
7572                         action_flags |= actions->type ==
7573                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
7574                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
7575                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
7576                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7577                                 modify_after_mirror = 1;
7578                         /*
7579                          * Even if the source and destination MAC addresses have
7580                          * overlap in the header with 4B alignment, the convert
7581                          * function will handle them separately and 4 SW actions
7582                          * will be created. And 2 actions will be added each
7583                          * time no matter how many bytes of address will be set.
7584                          */
7585                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
7586                         break;
7587                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
7588                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
7589                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
7590                                                                   actions,
7591                                                                   item_flags,
7592                                                                   error);
7593                         if (ret < 0)
7594                                 return ret;
7595                         /* Count all modify-header actions as one action. */
7596                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7597                                 ++actions_n;
7598                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7599                                 modify_after_mirror = 1;
7600                         action_flags |= actions->type ==
7601                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
7602                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
7603                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
7604                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
7605                         break;
7606                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
7607                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
7608                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
7609                                                                   actions,
7610                                                                   item_flags,
7611                                                                   error);
7612                         if (ret < 0)
7613                                 return ret;
7614                         if (item_ipv6_proto == IPPROTO_ICMPV6)
7615                                 return rte_flow_error_set(error, ENOTSUP,
7616                                         RTE_FLOW_ERROR_TYPE_ACTION,
7617                                         actions,
7618                                         "Can't change header "
7619                                         "with ICMPv6 proto");
7620                         /* Count all modify-header actions as one action. */
7621                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7622                                 ++actions_n;
7623                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7624                                 modify_after_mirror = 1;
7625                         action_flags |= actions->type ==
7626                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
7627                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
7628                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
7629                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
7630                         break;
7631                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
7632                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
7633                         ret = flow_dv_validate_action_modify_tp(action_flags,
7634                                                                 actions,
7635                                                                 item_flags,
7636                                                                 error);
7637                         if (ret < 0)
7638                                 return ret;
7639                         /* Count all modify-header actions as one action. */
7640                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7641                                 ++actions_n;
7642                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7643                                 modify_after_mirror = 1;
7644                         action_flags |= actions->type ==
7645                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
7646                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
7647                                                 MLX5_FLOW_ACTION_SET_TP_DST;
7648                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
7649                         break;
7650                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
7651                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
7652                         ret = flow_dv_validate_action_modify_ttl(action_flags,
7653                                                                  actions,
7654                                                                  item_flags,
7655                                                                  error);
7656                         if (ret < 0)
7657                                 return ret;
7658                         /* Count all modify-header actions as one action. */
7659                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7660                                 ++actions_n;
7661                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7662                                 modify_after_mirror = 1;
7663                         action_flags |= actions->type ==
7664                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
7665                                                 MLX5_FLOW_ACTION_SET_TTL :
7666                                                 MLX5_FLOW_ACTION_DEC_TTL;
7667                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
7668                         break;
7669                 case RTE_FLOW_ACTION_TYPE_JUMP:
7670                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
7671                                                            action_flags,
7672                                                            attr, external,
7673                                                            error);
7674                         if (ret)
7675                                 return ret;
7676                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7677                             fdb_mirror_limit)
7678                                 return rte_flow_error_set(error, EINVAL,
7679                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7680                                                   NULL,
7681                                                   "sample and jump action combination is not supported");
7682                         ++actions_n;
7683                         action_flags |= MLX5_FLOW_ACTION_JUMP;
7684                         break;
7685                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
7686                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
7687                         ret = flow_dv_validate_action_modify_tcp_seq
7688                                                                 (action_flags,
7689                                                                  actions,
7690                                                                  item_flags,
7691                                                                  error);
7692                         if (ret < 0)
7693                                 return ret;
7694                         /* Count all modify-header actions as one action. */
7695                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7696                                 ++actions_n;
7697                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7698                                 modify_after_mirror = 1;
7699                         action_flags |= actions->type ==
7700                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
7701                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
7702                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
7703                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
7704                         break;
7705                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
7706                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
7707                         ret = flow_dv_validate_action_modify_tcp_ack
7708                                                                 (action_flags,
7709                                                                  actions,
7710                                                                  item_flags,
7711                                                                  error);
7712                         if (ret < 0)
7713                                 return ret;
7714                         /* Count all modify-header actions as one action. */
7715                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7716                                 ++actions_n;
7717                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7718                                 modify_after_mirror = 1;
7719                         action_flags |= actions->type ==
7720                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
7721                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
7722                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
7723                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
7724                         break;
7725                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7726                         break;
7727                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
7728                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7729                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7730                         break;
7731                 case RTE_FLOW_ACTION_TYPE_METER:
7732                         ret = mlx5_flow_validate_action_meter(dev,
7733                                                               action_flags,
7734                                                               item_flags,
7735                                                               actions, attr,
7736                                                               port_id_item,
7737                                                               &def_policy,
7738                                                               error);
7739                         if (ret < 0)
7740                                 return ret;
7741                         action_flags |= MLX5_FLOW_ACTION_METER;
7742                         if (!def_policy)
7743                                 action_flags |=
7744                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
7745                         ++actions_n;
7746                         /* Meter action will add one more TAG action. */
7747                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7748                         break;
7749                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
7750                         if (!attr->transfer && !attr->group)
7751                                 return rte_flow_error_set(error, ENOTSUP,
7752                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7753                                                                            NULL,
7754                           "Shared ASO age action is not supported for group 0");
7755                         if (action_flags & MLX5_FLOW_ACTION_AGE)
7756                                 return rte_flow_error_set
7757                                                   (error, EINVAL,
7758                                                    RTE_FLOW_ERROR_TYPE_ACTION,
7759                                                    NULL,
7760                                                    "duplicate age actions set");
7761                         action_flags |= MLX5_FLOW_ACTION_AGE;
7762                         ++actions_n;
7763                         break;
7764                 case RTE_FLOW_ACTION_TYPE_AGE:
7765                         non_shared_age = actions->conf;
7766                         ret = flow_dv_validate_action_age(action_flags,
7767                                                           actions, dev,
7768                                                           error);
7769                         if (ret < 0)
7770                                 return ret;
7771                         /*
7772                          * Validate the regular AGE action (using counter)
7773                          * mutual exclusion with indirect counter actions.
7774                          */
7775                         if (!flow_hit_aso_supported(priv->sh, attr)) {
7776                                 if (shared_count)
7777                                         return rte_flow_error_set
7778                                                 (error, EINVAL,
7779                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7780                                                 NULL,
7781                                                 "old age and indirect count combination is not supported");
7782                                 if (sample_count)
7783                                         return rte_flow_error_set
7784                                                 (error, EINVAL,
7785                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7786                                                 NULL,
7787                                                 "old age action and count must be in the same sub flow");
7788                         }
7789                         action_flags |= MLX5_FLOW_ACTION_AGE;
7790                         ++actions_n;
7791                         break;
7792                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
7793                         ret = flow_dv_validate_action_modify_ipv4_dscp
7794                                                          (action_flags,
7795                                                           actions,
7796                                                           item_flags,
7797                                                           error);
7798                         if (ret < 0)
7799                                 return ret;
7800                         /* Count all modify-header actions as one action. */
7801                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7802                                 ++actions_n;
7803                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7804                                 modify_after_mirror = 1;
7805                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
7806                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7807                         break;
7808                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
7809                         ret = flow_dv_validate_action_modify_ipv6_dscp
7810                                                                 (action_flags,
7811                                                                  actions,
7812                                                                  item_flags,
7813                                                                  error);
7814                         if (ret < 0)
7815                                 return ret;
7816                         /* Count all modify-header actions as one action. */
7817                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7818                                 ++actions_n;
7819                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7820                                 modify_after_mirror = 1;
7821                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
7822                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7823                         break;
7824                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
7825                         ret = flow_dv_validate_action_sample(&action_flags,
7826                                                              actions, dev,
7827                                                              attr, item_flags,
7828                                                              rss, &sample_rss,
7829                                                              &sample_count,
7830                                                              &fdb_mirror_limit,
7831                                                              error);
7832                         if (ret < 0)
7833                                 return ret;
7834                         if ((action_flags & MLX5_FLOW_ACTION_SET_TAG) &&
7835                             tag_id == 0 && priv->mtr_color_reg == REG_NON)
7836                                 return rte_flow_error_set(error, EINVAL,
7837                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7838                                         "sample after tag action causes metadata tag index 0 corruption");
7839                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
7840                         ++actions_n;
7841                         break;
7842                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
7843                         ret = flow_dv_validate_action_modify_field(dev,
7844                                                                    action_flags,
7845                                                                    actions,
7846                                                                    attr,
7847                                                                    error);
7848                         if (ret < 0)
7849                                 return ret;
7850                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7851                                 modify_after_mirror = 1;
7852                         /* Count all modify-header actions as one action. */
7853                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7854                                 ++actions_n;
7855                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
7856                         rw_act_num += ret;
7857                         break;
7858                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
7859                         ret = flow_dv_validate_action_aso_ct(dev, action_flags,
7860                                                              item_flags, attr,
7861                                                              error);
7862                         if (ret < 0)
7863                                 return ret;
7864                         action_flags |= MLX5_FLOW_ACTION_CT;
7865                         break;
7866                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
7867                         /* tunnel offload action was processed before
7868                          * list it here as a supported type
7869                          */
7870                         break;
7871                 default:
7872                         return rte_flow_error_set(error, ENOTSUP,
7873                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7874                                                   actions,
7875                                                   "action not supported");
7876                 }
7877         }
7878         /*
7879          * Validate actions in flow rules
7880          * - Explicit decap action is prohibited by the tunnel offload API.
7881          * - Drop action in tunnel steer rule is prohibited by the API.
7882          * - Application cannot use MARK action because it's value can mask
7883          *   tunnel default miss notification.
7884          * - JUMP in tunnel match rule has no support in current PMD
7885          *   implementation.
7886          * - TAG & META are reserved for future uses.
7887          */
7888         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
7889                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
7890                                             MLX5_FLOW_ACTION_MARK     |
7891                                             MLX5_FLOW_ACTION_SET_TAG  |
7892                                             MLX5_FLOW_ACTION_SET_META |
7893                                             MLX5_FLOW_ACTION_DROP;
7894
7895                 if (action_flags & bad_actions_mask)
7896                         return rte_flow_error_set
7897                                         (error, EINVAL,
7898                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7899                                         "Invalid RTE action in tunnel "
7900                                         "set decap rule");
7901                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
7902                         return rte_flow_error_set
7903                                         (error, EINVAL,
7904                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7905                                         "tunnel set decap rule must terminate "
7906                                         "with JUMP");
7907                 if (!attr->ingress)
7908                         return rte_flow_error_set
7909                                         (error, EINVAL,
7910                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7911                                         "tunnel flows for ingress traffic only");
7912         }
7913         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
7914                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
7915                                             MLX5_FLOW_ACTION_MARK    |
7916                                             MLX5_FLOW_ACTION_SET_TAG |
7917                                             MLX5_FLOW_ACTION_SET_META;
7918
7919                 if (action_flags & bad_actions_mask)
7920                         return rte_flow_error_set
7921                                         (error, EINVAL,
7922                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7923                                         "Invalid RTE action in tunnel "
7924                                         "set match rule");
7925         }
7926         /*
7927          * Validate the drop action mutual exclusion with other actions.
7928          * Drop action is mutually-exclusive with any other action, except for
7929          * Count action.
7930          * Drop action compatibility with tunnel offload was already validated.
7931          */
7932         if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH |
7933                             MLX5_FLOW_ACTION_TUNNEL_MATCH));
7934         else if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
7935             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
7936                 return rte_flow_error_set(error, EINVAL,
7937                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7938                                           "Drop action is mutually-exclusive "
7939                                           "with any other action, except for "
7940                                           "Count action");
7941         /* Eswitch has few restrictions on using items and actions */
7942         if (attr->transfer) {
7943                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7944                     action_flags & MLX5_FLOW_ACTION_FLAG)
7945                         return rte_flow_error_set(error, ENOTSUP,
7946                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7947                                                   NULL,
7948                                                   "unsupported action FLAG");
7949                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7950                     action_flags & MLX5_FLOW_ACTION_MARK)
7951                         return rte_flow_error_set(error, ENOTSUP,
7952                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7953                                                   NULL,
7954                                                   "unsupported action MARK");
7955                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
7956                         return rte_flow_error_set(error, ENOTSUP,
7957                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7958                                                   NULL,
7959                                                   "unsupported action QUEUE");
7960                 if (action_flags & MLX5_FLOW_ACTION_RSS)
7961                         return rte_flow_error_set(error, ENOTSUP,
7962                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7963                                                   NULL,
7964                                                   "unsupported action RSS");
7965                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
7966                         return rte_flow_error_set(error, EINVAL,
7967                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7968                                                   actions,
7969                                                   "no fate action is found");
7970         } else {
7971                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
7972                         return rte_flow_error_set(error, EINVAL,
7973                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7974                                                   actions,
7975                                                   "no fate action is found");
7976         }
7977         /*
7978          * Continue validation for Xcap and VLAN actions.
7979          * If hairpin is working in explicit TX rule mode, there is no actions
7980          * splitting and the validation of hairpin ingress flow should be the
7981          * same as other standard flows.
7982          */
7983         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
7984                              MLX5_FLOW_VLAN_ACTIONS)) &&
7985             (queue_index == 0xFFFF || !mlx5_rxq_is_hairpin(dev, queue_index) ||
7986              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
7987              conf->tx_explicit != 0))) {
7988                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
7989                     MLX5_FLOW_XCAP_ACTIONS)
7990                         return rte_flow_error_set(error, ENOTSUP,
7991                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7992                                                   NULL, "encap and decap "
7993                                                   "combination aren't supported");
7994                 /* Push VLAN is not supported in ingress except for NICs newer than CX5. */
7995                 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) {
7996                         struct mlx5_dev_ctx_shared *sh = priv->sh;
7997                         bool direction_error = false;
7998
7999                         if (attr->transfer) {
8000                                 bool fdb_tx = priv->representor_id != UINT16_MAX;
8001                                 bool is_cx5 = sh->steering_format_version ==
8002                                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
8003
8004                                 if (!fdb_tx && is_cx5)
8005                                         direction_error = true;
8006                         } else if (attr->ingress) {
8007                                 direction_error = true;
8008                         }
8009                         if (direction_error)
8010                                 return rte_flow_error_set(error, ENOTSUP,
8011                                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
8012                                                           NULL,
8013                                                           "push VLAN action not supported "
8014                                                           "for ingress");
8015                 }
8016                 if (!attr->transfer && attr->ingress) {
8017                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
8018                                 return rte_flow_error_set
8019                                                 (error, ENOTSUP,
8020                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8021                                                  NULL, "encap is not supported"
8022                                                  " for ingress traffic");
8023                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
8024                                         MLX5_FLOW_VLAN_ACTIONS)
8025                                 return rte_flow_error_set
8026                                                 (error, ENOTSUP,
8027                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8028                                                  NULL, "no support for "
8029                                                  "multiple VLAN actions");
8030                 }
8031         }
8032         if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) {
8033                 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS &
8034                         ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) &&
8035                         attr->ingress)
8036                         return rte_flow_error_set
8037                                 (error, ENOTSUP,
8038                                 RTE_FLOW_ERROR_TYPE_ACTION,
8039                                 NULL, "fate action not supported for "
8040                                 "meter with policy");
8041                 if (attr->egress) {
8042                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
8043                                 return rte_flow_error_set
8044                                         (error, ENOTSUP,
8045                                         RTE_FLOW_ERROR_TYPE_ACTION,
8046                                         NULL, "modify header action in egress "
8047                                         "cannot be done before meter action");
8048                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
8049                                 return rte_flow_error_set
8050                                         (error, ENOTSUP,
8051                                         RTE_FLOW_ERROR_TYPE_ACTION,
8052                                         NULL, "encap action in egress "
8053                                         "cannot be done before meter action");
8054                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8055                                 return rte_flow_error_set
8056                                         (error, ENOTSUP,
8057                                         RTE_FLOW_ERROR_TYPE_ACTION,
8058                                         NULL, "push vlan action in egress "
8059                                         "cannot be done before meter action");
8060                 }
8061         }
8062         /*
8063          * Only support one ASO action in a single flow rule.
8064          * non-shared AGE + counter will fallback to use HW counter, no ASO hit object.
8065          * Group 0 uses HW counter for AGE too even if no counter action.
8066          */
8067         aso_mask = (action_flags & MLX5_FLOW_ACTION_METER && priv->sh->meter_aso_en) << 2 |
8068                    (action_flags & MLX5_FLOW_ACTION_CT && priv->sh->ct_aso_en) << 1 |
8069                    (action_flags & MLX5_FLOW_ACTION_AGE &&
8070                     !(non_shared_age && count) &&
8071                     (attr->group || (attr->transfer && priv->fdb_def_rule)) &&
8072                     priv->sh->flow_hit_aso_en);
8073         if (__builtin_popcountl(aso_mask) > 1)
8074                 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8075                                           NULL, "unsupported combining AGE, METER, CT ASO actions in a single rule");
8076         /*
8077          * Hairpin flow will add one more TAG action in TX implicit mode.
8078          * In TX explicit mode, there will be no hairpin flow ID.
8079          */
8080         if (hairpin > 0)
8081                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
8082         /* extra metadata enabled: one more TAG action will be add. */
8083         if (dev_conf->dv_flow_en &&
8084             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
8085             mlx5_flow_ext_mreg_supported(dev))
8086                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
8087         if (rw_act_num >
8088                         flow_dv_modify_hdr_action_max(dev, is_root)) {
8089                 return rte_flow_error_set(error, ENOTSUP,
8090                                           RTE_FLOW_ERROR_TYPE_ACTION,
8091                                           NULL, "too many header modify"
8092                                           " actions to support");
8093         }
8094         /* Eswitch egress mirror and modify flow has limitation on CX5 */
8095         if (fdb_mirror_limit && modify_after_mirror)
8096                 return rte_flow_error_set(error, EINVAL,
8097                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
8098                                 "sample before modify action is not supported");
8099         /*
8100          * Validation the NIC Egress flow on representor, except implicit
8101          * hairpin default egress flow with TX_QUEUE item, other flows not
8102          * work due to metadata regC0 mismatch.
8103          */
8104         if ((!attr->transfer && attr->egress) && priv->representor &&
8105             !(item_flags & MLX5_FLOW_ITEM_TX_QUEUE))
8106                 return rte_flow_error_set(error, EINVAL,
8107                                           RTE_FLOW_ERROR_TYPE_ITEM,
8108                                           NULL,
8109                                           "NIC egress rules on representors"
8110                                           " is not supported");
8111         return 0;
8112 }
8113
8114 /**
8115  * Internal preparation function. Allocates the DV flow size,
8116  * this size is constant.
8117  *
8118  * @param[in] dev
8119  *   Pointer to the rte_eth_dev structure.
8120  * @param[in] attr
8121  *   Pointer to the flow attributes.
8122  * @param[in] items
8123  *   Pointer to the list of items.
8124  * @param[in] actions
8125  *   Pointer to the list of actions.
8126  * @param[out] error
8127  *   Pointer to the error structure.
8128  *
8129  * @return
8130  *   Pointer to mlx5_flow object on success,
8131  *   otherwise NULL and rte_errno is set.
8132  */
8133 static struct mlx5_flow *
8134 flow_dv_prepare(struct rte_eth_dev *dev,
8135                 const struct rte_flow_attr *attr __rte_unused,
8136                 const struct rte_flow_item items[] __rte_unused,
8137                 const struct rte_flow_action actions[] __rte_unused,
8138                 struct rte_flow_error *error)
8139 {
8140         uint32_t handle_idx = 0;
8141         struct mlx5_flow *dev_flow;
8142         struct mlx5_flow_handle *dev_handle;
8143         struct mlx5_priv *priv = dev->data->dev_private;
8144         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
8145
8146         MLX5_ASSERT(wks);
8147         wks->skip_matcher_reg = 0;
8148         wks->policy = NULL;
8149         wks->final_policy = NULL;
8150         /* In case of corrupting the memory. */
8151         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
8152                 rte_flow_error_set(error, ENOSPC,
8153                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8154                                    "not free temporary device flow");
8155                 return NULL;
8156         }
8157         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8158                                    &handle_idx);
8159         if (!dev_handle) {
8160                 rte_flow_error_set(error, ENOMEM,
8161                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8162                                    "not enough memory to create flow handle");
8163                 return NULL;
8164         }
8165         MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows));
8166         dev_flow = &wks->flows[wks->flow_idx++];
8167         memset(dev_flow, 0, sizeof(*dev_flow));
8168         dev_flow->handle = dev_handle;
8169         dev_flow->handle_idx = handle_idx;
8170         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
8171         dev_flow->ingress = attr->ingress;
8172         dev_flow->dv.transfer = attr->transfer;
8173         return dev_flow;
8174 }
8175
8176 #ifdef RTE_LIBRTE_MLX5_DEBUG
8177 /**
8178  * Sanity check for match mask and value. Similar to check_valid_spec() in
8179  * kernel driver. If unmasked bit is present in value, it returns failure.
8180  *
8181  * @param match_mask
8182  *   pointer to match mask buffer.
8183  * @param match_value
8184  *   pointer to match value buffer.
8185  *
8186  * @return
8187  *   0 if valid, -EINVAL otherwise.
8188  */
8189 static int
8190 flow_dv_check_valid_spec(void *match_mask, void *match_value)
8191 {
8192         uint8_t *m = match_mask;
8193         uint8_t *v = match_value;
8194         unsigned int i;
8195
8196         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
8197                 if (v[i] & ~m[i]) {
8198                         DRV_LOG(ERR,
8199                                 "match_value differs from match_criteria"
8200                                 " %p[%u] != %p[%u]",
8201                                 match_value, i, match_mask, i);
8202                         return -EINVAL;
8203                 }
8204         }
8205         return 0;
8206 }
8207 #endif
8208
8209 /**
8210  * Add match of ip_version.
8211  *
8212  * @param[in] group
8213  *   Flow group.
8214  * @param[in] headers_v
8215  *   Values header pointer.
8216  * @param[in] headers_m
8217  *   Masks header pointer.
8218  * @param[in] ip_version
8219  *   The IP version to set.
8220  */
8221 static inline void
8222 flow_dv_set_match_ip_version(uint32_t group,
8223                              void *headers_v,
8224                              void *headers_m,
8225                              uint8_t ip_version)
8226 {
8227         if (group == 0)
8228                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
8229         else
8230                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
8231                          ip_version);
8232         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
8233         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
8234         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
8235 }
8236
8237 /**
8238  * Add Ethernet item to matcher and to the value.
8239  *
8240  * @param[in, out] matcher
8241  *   Flow matcher.
8242  * @param[in, out] key
8243  *   Flow matcher value.
8244  * @param[in] item
8245  *   Flow pattern to translate.
8246  * @param[in] inner
8247  *   Item is inner pattern.
8248  */
8249 static void
8250 flow_dv_translate_item_eth(void *matcher, void *key,
8251                            const struct rte_flow_item *item, int inner,
8252                            uint32_t group)
8253 {
8254         const struct rte_flow_item_eth *eth_m = item->mask;
8255         const struct rte_flow_item_eth *eth_v = item->spec;
8256         const struct rte_flow_item_eth nic_mask = {
8257                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8258                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8259                 .type = RTE_BE16(0xffff),
8260                 .has_vlan = 0,
8261         };
8262         void *hdrs_m;
8263         void *hdrs_v;
8264         char *l24_v;
8265         unsigned int i;
8266
8267         if (!eth_v)
8268                 return;
8269         if (!eth_m)
8270                 eth_m = &nic_mask;
8271         if (inner) {
8272                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8273                                          inner_headers);
8274                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8275         } else {
8276                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8277                                          outer_headers);
8278                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8279         }
8280         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
8281                &eth_m->dst, sizeof(eth_m->dst));
8282         /* The value must be in the range of the mask. */
8283         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
8284         for (i = 0; i < sizeof(eth_m->dst); ++i)
8285                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
8286         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
8287                &eth_m->src, sizeof(eth_m->src));
8288         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
8289         /* The value must be in the range of the mask. */
8290         for (i = 0; i < sizeof(eth_m->dst); ++i)
8291                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
8292         /*
8293          * HW supports match on one Ethertype, the Ethertype following the last
8294          * VLAN tag of the packet (see PRM).
8295          * Set match on ethertype only if ETH header is not followed by VLAN.
8296          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8297          * ethertype, and use ip_version field instead.
8298          * eCPRI over Ether layer will use type value 0xAEFE.
8299          */
8300         if (eth_m->type == 0xFFFF) {
8301                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
8302                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8303                 switch (eth_v->type) {
8304                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8305                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8306                         return;
8307                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
8308                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8309                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8310                         return;
8311                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8312                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8313                         return;
8314                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8315                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8316                         return;
8317                 default:
8318                         break;
8319                 }
8320         }
8321         if (eth_m->has_vlan) {
8322                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8323                 if (eth_v->has_vlan) {
8324                         /*
8325                          * Here, when also has_more_vlan field in VLAN item is
8326                          * not set, only single-tagged packets will be matched.
8327                          */
8328                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8329                         return;
8330                 }
8331         }
8332         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8333                  rte_be_to_cpu_16(eth_m->type));
8334         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
8335         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
8336 }
8337
8338 /**
8339  * Add VLAN item to matcher and to the value.
8340  *
8341  * @param[in, out] dev_flow
8342  *   Flow descriptor.
8343  * @param[in, out] matcher
8344  *   Flow matcher.
8345  * @param[in, out] key
8346  *   Flow matcher value.
8347  * @param[in] item
8348  *   Flow pattern to translate.
8349  * @param[in] inner
8350  *   Item is inner pattern.
8351  */
8352 static void
8353 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
8354                             void *matcher, void *key,
8355                             const struct rte_flow_item *item,
8356                             int inner, uint32_t group)
8357 {
8358         const struct rte_flow_item_vlan *vlan_m = item->mask;
8359         const struct rte_flow_item_vlan *vlan_v = item->spec;
8360         void *hdrs_m;
8361         void *hdrs_v;
8362         uint16_t tci_m;
8363         uint16_t tci_v;
8364
8365         if (inner) {
8366                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8367                                          inner_headers);
8368                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8369         } else {
8370                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8371                                          outer_headers);
8372                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8373                 /*
8374                  * This is workaround, masks are not supported,
8375                  * and pre-validated.
8376                  */
8377                 if (vlan_v)
8378                         dev_flow->handle->vf_vlan.tag =
8379                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
8380         }
8381         /*
8382          * When VLAN item exists in flow, mark packet as tagged,
8383          * even if TCI is not specified.
8384          */
8385         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
8386                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8387                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8388         }
8389         if (!vlan_v)
8390                 return;
8391         if (!vlan_m)
8392                 vlan_m = &rte_flow_item_vlan_mask;
8393         tci_m = rte_be_to_cpu_16(vlan_m->tci);
8394         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
8395         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
8396         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
8397         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
8398         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
8399         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
8400         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
8401         /*
8402          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8403          * ethertype, and use ip_version field instead.
8404          */
8405         if (vlan_m->inner_type == 0xFFFF) {
8406                 switch (vlan_v->inner_type) {
8407                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8408                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8409                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8410                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8411                         return;
8412                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8413                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8414                         return;
8415                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8416                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8417                         return;
8418                 default:
8419                         break;
8420                 }
8421         }
8422         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
8423                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8424                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8425                 /* Only one vlan_tag bit can be set. */
8426                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8427                 return;
8428         }
8429         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8430                  rte_be_to_cpu_16(vlan_m->inner_type));
8431         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
8432                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
8433 }
8434
8435 /**
8436  * Add IPV4 item to matcher and to the value.
8437  *
8438  * @param[in, out] matcher
8439  *   Flow matcher.
8440  * @param[in, out] key
8441  *   Flow matcher value.
8442  * @param[in] item
8443  *   Flow pattern to translate.
8444  * @param[in] inner
8445  *   Item is inner pattern.
8446  * @param[in] group
8447  *   The group to insert the rule.
8448  */
8449 static void
8450 flow_dv_translate_item_ipv4(void *matcher, void *key,
8451                             const struct rte_flow_item *item,
8452                             int inner, uint32_t group)
8453 {
8454         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
8455         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
8456         const struct rte_flow_item_ipv4 nic_mask = {
8457                 .hdr = {
8458                         .src_addr = RTE_BE32(0xffffffff),
8459                         .dst_addr = RTE_BE32(0xffffffff),
8460                         .type_of_service = 0xff,
8461                         .next_proto_id = 0xff,
8462                         .time_to_live = 0xff,
8463                 },
8464         };
8465         void *headers_m;
8466         void *headers_v;
8467         char *l24_m;
8468         char *l24_v;
8469         uint8_t tos, ihl_m, ihl_v;
8470
8471         if (inner) {
8472                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8473                                          inner_headers);
8474                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8475         } else {
8476                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8477                                          outer_headers);
8478                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8479         }
8480         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
8481         if (!ipv4_v)
8482                 return;
8483         if (!ipv4_m)
8484                 ipv4_m = &nic_mask;
8485         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8486                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8487         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8488                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8489         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
8490         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
8491         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8492                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8493         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8494                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8495         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
8496         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
8497         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
8498         ihl_m = ipv4_m->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8499         ihl_v = ipv4_v->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8500         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_ihl, ihl_m);
8501         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_ihl, ihl_m & ihl_v);
8502         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
8503                  ipv4_m->hdr.type_of_service);
8504         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
8505         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
8506                  ipv4_m->hdr.type_of_service >> 2);
8507         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
8508         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8509                  ipv4_m->hdr.next_proto_id);
8510         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8511                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
8512         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8513                  ipv4_m->hdr.time_to_live);
8514         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8515                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
8516         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8517                  !!(ipv4_m->hdr.fragment_offset));
8518         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8519                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
8520 }
8521
8522 /**
8523  * Add IPV6 item to matcher and to the value.
8524  *
8525  * @param[in, out] matcher
8526  *   Flow matcher.
8527  * @param[in, out] key
8528  *   Flow matcher value.
8529  * @param[in] item
8530  *   Flow pattern to translate.
8531  * @param[in] inner
8532  *   Item is inner pattern.
8533  * @param[in] group
8534  *   The group to insert the rule.
8535  */
8536 static void
8537 flow_dv_translate_item_ipv6(void *matcher, void *key,
8538                             const struct rte_flow_item *item,
8539                             int inner, uint32_t group)
8540 {
8541         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
8542         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
8543         const struct rte_flow_item_ipv6 nic_mask = {
8544                 .hdr = {
8545                         .src_addr =
8546                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8547                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8548                         .dst_addr =
8549                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8550                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8551                         .vtc_flow = RTE_BE32(0xffffffff),
8552                         .proto = 0xff,
8553                         .hop_limits = 0xff,
8554                 },
8555         };
8556         void *headers_m;
8557         void *headers_v;
8558         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8559         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8560         char *l24_m;
8561         char *l24_v;
8562         uint32_t vtc_m;
8563         uint32_t vtc_v;
8564         int i;
8565         int size;
8566
8567         if (inner) {
8568                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8569                                          inner_headers);
8570                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8571         } else {
8572                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8573                                          outer_headers);
8574                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8575         }
8576         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
8577         if (!ipv6_v)
8578                 return;
8579         if (!ipv6_m)
8580                 ipv6_m = &nic_mask;
8581         size = sizeof(ipv6_m->hdr.dst_addr);
8582         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8583                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8584         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8585                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8586         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
8587         for (i = 0; i < size; ++i)
8588                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
8589         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8590                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8591         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8592                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8593         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
8594         for (i = 0; i < size; ++i)
8595                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
8596         /* TOS. */
8597         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
8598         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
8599         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
8600         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
8601         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
8602         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
8603         /* Label. */
8604         if (inner) {
8605                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
8606                          vtc_m);
8607                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
8608                          vtc_v);
8609         } else {
8610                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
8611                          vtc_m);
8612                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
8613                          vtc_v);
8614         }
8615         /* Protocol. */
8616         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8617                  ipv6_m->hdr.proto);
8618         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8619                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
8620         /* Hop limit. */
8621         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8622                  ipv6_m->hdr.hop_limits);
8623         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8624                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
8625         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8626                  !!(ipv6_m->has_frag_ext));
8627         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8628                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
8629 }
8630
8631 /**
8632  * Add IPV6 fragment extension item to matcher and to the value.
8633  *
8634  * @param[in, out] matcher
8635  *   Flow matcher.
8636  * @param[in, out] key
8637  *   Flow matcher value.
8638  * @param[in] item
8639  *   Flow pattern to translate.
8640  * @param[in] inner
8641  *   Item is inner pattern.
8642  */
8643 static void
8644 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
8645                                      const struct rte_flow_item *item,
8646                                      int inner)
8647 {
8648         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
8649         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
8650         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
8651                 .hdr = {
8652                         .next_header = 0xff,
8653                         .frag_data = RTE_BE16(0xffff),
8654                 },
8655         };
8656         void *headers_m;
8657         void *headers_v;
8658
8659         if (inner) {
8660                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8661                                          inner_headers);
8662                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8663         } else {
8664                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8665                                          outer_headers);
8666                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8667         }
8668         /* IPv6 fragment extension item exists, so packet is IP fragment. */
8669         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
8670         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
8671         if (!ipv6_frag_ext_v)
8672                 return;
8673         if (!ipv6_frag_ext_m)
8674                 ipv6_frag_ext_m = &nic_mask;
8675         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8676                  ipv6_frag_ext_m->hdr.next_header);
8677         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8678                  ipv6_frag_ext_v->hdr.next_header &
8679                  ipv6_frag_ext_m->hdr.next_header);
8680 }
8681
8682 /**
8683  * Add TCP item to matcher and to the value.
8684  *
8685  * @param[in, out] matcher
8686  *   Flow matcher.
8687  * @param[in, out] key
8688  *   Flow matcher value.
8689  * @param[in] item
8690  *   Flow pattern to translate.
8691  * @param[in] inner
8692  *   Item is inner pattern.
8693  */
8694 static void
8695 flow_dv_translate_item_tcp(void *matcher, void *key,
8696                            const struct rte_flow_item *item,
8697                            int inner)
8698 {
8699         const struct rte_flow_item_tcp *tcp_m = item->mask;
8700         const struct rte_flow_item_tcp *tcp_v = item->spec;
8701         void *headers_m;
8702         void *headers_v;
8703
8704         if (inner) {
8705                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8706                                          inner_headers);
8707                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8708         } else {
8709                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8710                                          outer_headers);
8711                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8712         }
8713         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8714         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
8715         if (!tcp_v)
8716                 return;
8717         if (!tcp_m)
8718                 tcp_m = &rte_flow_item_tcp_mask;
8719         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
8720                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
8721         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
8722                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
8723         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
8724                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
8725         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
8726                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
8727         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
8728                  tcp_m->hdr.tcp_flags);
8729         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
8730                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
8731 }
8732
8733 /**
8734  * Add UDP item to matcher and to the value.
8735  *
8736  * @param[in, out] matcher
8737  *   Flow matcher.
8738  * @param[in, out] key
8739  *   Flow matcher value.
8740  * @param[in] item
8741  *   Flow pattern to translate.
8742  * @param[in] inner
8743  *   Item is inner pattern.
8744  */
8745 static void
8746 flow_dv_translate_item_udp(void *matcher, void *key,
8747                            const struct rte_flow_item *item,
8748                            int inner)
8749 {
8750         const struct rte_flow_item_udp *udp_m = item->mask;
8751         const struct rte_flow_item_udp *udp_v = item->spec;
8752         void *headers_m;
8753         void *headers_v;
8754
8755         if (inner) {
8756                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8757                                          inner_headers);
8758                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8759         } else {
8760                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8761                                          outer_headers);
8762                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8763         }
8764         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8765         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
8766         if (!udp_v)
8767                 return;
8768         if (!udp_m)
8769                 udp_m = &rte_flow_item_udp_mask;
8770         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
8771                  rte_be_to_cpu_16(udp_m->hdr.src_port));
8772         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
8773                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
8774         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
8775                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
8776         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
8777                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
8778 }
8779
8780 /**
8781  * Add GRE optional Key item to matcher and to the value.
8782  *
8783  * @param[in, out] matcher
8784  *   Flow matcher.
8785  * @param[in, out] key
8786  *   Flow matcher value.
8787  * @param[in] item
8788  *   Flow pattern to translate.
8789  * @param[in] inner
8790  *   Item is inner pattern.
8791  */
8792 static void
8793 flow_dv_translate_item_gre_key(void *matcher, void *key,
8794                                    const struct rte_flow_item *item)
8795 {
8796         const rte_be32_t *key_m = item->mask;
8797         const rte_be32_t *key_v = item->spec;
8798         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8799         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8800         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
8801
8802         /* GRE K bit must be on and should already be validated */
8803         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
8804         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
8805         if (!key_v)
8806                 return;
8807         if (!key_m)
8808                 key_m = &gre_key_default_mask;
8809         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
8810                  rte_be_to_cpu_32(*key_m) >> 8);
8811         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
8812                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
8813         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
8814                  rte_be_to_cpu_32(*key_m) & 0xFF);
8815         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
8816                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
8817 }
8818
8819 /**
8820  * Add GRE item to matcher and to the value.
8821  *
8822  * @param[in, out] matcher
8823  *   Flow matcher.
8824  * @param[in, out] key
8825  *   Flow matcher value.
8826  * @param[in] item
8827  *   Flow pattern to translate.
8828  * @param[in] pattern_flags
8829  *   Accumulated pattern flags.
8830  */
8831 static void
8832 flow_dv_translate_item_gre(void *matcher, void *key,
8833                            const struct rte_flow_item *item,
8834                            uint64_t pattern_flags)
8835 {
8836         static const struct rte_flow_item_gre empty_gre = {0,};
8837         const struct rte_flow_item_gre *gre_m = item->mask;
8838         const struct rte_flow_item_gre *gre_v = item->spec;
8839         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
8840         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8841         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8842         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8843         struct {
8844                 union {
8845                         __extension__
8846                         struct {
8847                                 uint16_t version:3;
8848                                 uint16_t rsvd0:9;
8849                                 uint16_t s_present:1;
8850                                 uint16_t k_present:1;
8851                                 uint16_t rsvd_bit1:1;
8852                                 uint16_t c_present:1;
8853                         };
8854                         uint16_t value;
8855                 };
8856         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
8857         uint16_t protocol_m, protocol_v;
8858
8859         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8860         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
8861         if (!gre_v) {
8862                 gre_v = &empty_gre;
8863                 gre_m = &empty_gre;
8864         } else {
8865                 if (!gre_m)
8866                         gre_m = &rte_flow_item_gre_mask;
8867         }
8868         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
8869         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
8870         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
8871                  gre_crks_rsvd0_ver_m.c_present);
8872         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
8873                  gre_crks_rsvd0_ver_v.c_present &
8874                  gre_crks_rsvd0_ver_m.c_present);
8875         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
8876                  gre_crks_rsvd0_ver_m.k_present);
8877         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
8878                  gre_crks_rsvd0_ver_v.k_present &
8879                  gre_crks_rsvd0_ver_m.k_present);
8880         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
8881                  gre_crks_rsvd0_ver_m.s_present);
8882         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
8883                  gre_crks_rsvd0_ver_v.s_present &
8884                  gre_crks_rsvd0_ver_m.s_present);
8885         protocol_m = rte_be_to_cpu_16(gre_m->protocol);
8886         protocol_v = rte_be_to_cpu_16(gre_v->protocol);
8887         if (!protocol_m) {
8888                 /* Force next protocol to prevent matchers duplication */
8889                 protocol_v = mlx5_translate_tunnel_etypes(pattern_flags);
8890                 if (protocol_v)
8891                         protocol_m = 0xFFFF;
8892         }
8893         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, protocol_m);
8894         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
8895                  protocol_m & protocol_v);
8896 }
8897
8898 /**
8899  * Add GRE optional items to matcher and to the value.
8900  *
8901  * @param[in, out] matcher
8902  *   Flow matcher.
8903  * @param[in, out] key
8904  *   Flow matcher value.
8905  * @param[in] item
8906  *   Flow pattern to translate.
8907  * @param[in] gre_item
8908  *   Pointer to gre_item.
8909  * @param[in] pattern_flags
8910  *   Accumulated pattern flags.
8911  */
8912 static void
8913 flow_dv_translate_item_gre_option(void *matcher, void *key,
8914                                   const struct rte_flow_item *item,
8915                                   const struct rte_flow_item *gre_item,
8916                                   uint64_t pattern_flags)
8917 {
8918         const struct rte_flow_item_gre_opt *option_m = item->mask;
8919         const struct rte_flow_item_gre_opt *option_v = item->spec;
8920         const struct rte_flow_item_gre *gre_m = gre_item->mask;
8921         const struct rte_flow_item_gre *gre_v = gre_item->spec;
8922         static const struct rte_flow_item_gre empty_gre = {0};
8923         struct rte_flow_item gre_key_item;
8924         uint16_t c_rsvd0_ver_m, c_rsvd0_ver_v;
8925         uint16_t protocol_m, protocol_v;
8926         void *misc5_m;
8927         void *misc5_v;
8928
8929         /*
8930          * If only match key field, keep using misc for matching.
8931          * If need to match checksum or sequence, using misc5 and do
8932          * not need using misc.
8933          */
8934         if (!(option_m->sequence.sequence ||
8935               option_m->checksum_rsvd.checksum)) {
8936                 flow_dv_translate_item_gre(matcher, key, gre_item,
8937                                            pattern_flags);
8938                 gre_key_item.spec = &option_v->key.key;
8939                 gre_key_item.mask = &option_m->key.key;
8940                 flow_dv_translate_item_gre_key(matcher, key, &gre_key_item);
8941                 return;
8942         }
8943         if (!gre_v) {
8944                 gre_v = &empty_gre;
8945                 gre_m = &empty_gre;
8946         } else {
8947                 if (!gre_m)
8948                         gre_m = &rte_flow_item_gre_mask;
8949         }
8950         protocol_v = gre_v->protocol;
8951         protocol_m = gre_m->protocol;
8952         if (!protocol_m) {
8953                 /* Force next protocol to prevent matchers duplication */
8954                 uint16_t ether_type =
8955                         mlx5_translate_tunnel_etypes(pattern_flags);
8956                 if (ether_type) {
8957                         protocol_v = rte_be_to_cpu_16(ether_type);
8958                         protocol_m = UINT16_MAX;
8959                 }
8960         }
8961         c_rsvd0_ver_v = gre_v->c_rsvd0_ver;
8962         c_rsvd0_ver_m = gre_m->c_rsvd0_ver;
8963         if (option_m->sequence.sequence) {
8964                 c_rsvd0_ver_v |= RTE_BE16(0x1000);
8965                 c_rsvd0_ver_m |= RTE_BE16(0x1000);
8966         }
8967         if (option_m->key.key) {
8968                 c_rsvd0_ver_v |= RTE_BE16(0x2000);
8969                 c_rsvd0_ver_m |= RTE_BE16(0x2000);
8970         }
8971         if (option_m->checksum_rsvd.checksum) {
8972                 c_rsvd0_ver_v |= RTE_BE16(0x8000);
8973                 c_rsvd0_ver_m |= RTE_BE16(0x8000);
8974         }
8975         /*
8976          * Hardware parses GRE optional field into the fixed location,
8977          * do not need to adjust the tunnel dword indices.
8978          */
8979         misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
8980         misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
8981         MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_0,
8982                  rte_be_to_cpu_32((c_rsvd0_ver_v | protocol_v << 16) &
8983                                   (c_rsvd0_ver_m | protocol_m << 16)));
8984         MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_0,
8985                  rte_be_to_cpu_32(c_rsvd0_ver_m | protocol_m << 16));
8986         MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_1,
8987                  rte_be_to_cpu_32(option_v->checksum_rsvd.checksum &
8988                                   option_m->checksum_rsvd.checksum));
8989         MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_1,
8990                  rte_be_to_cpu_32(option_m->checksum_rsvd.checksum));
8991         MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_2,
8992                  rte_be_to_cpu_32(option_v->key.key & option_m->key.key));
8993         MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_2,
8994                  rte_be_to_cpu_32(option_m->key.key));
8995         MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_3,
8996                  rte_be_to_cpu_32(option_v->sequence.sequence &
8997                                   option_m->sequence.sequence));
8998         MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_3,
8999                  rte_be_to_cpu_32(option_m->sequence.sequence));
9000 }
9001
9002 /**
9003  * Add NVGRE item to matcher and to the value.
9004  *
9005  * @param[in, out] matcher
9006  *   Flow matcher.
9007  * @param[in, out] key
9008  *   Flow matcher value.
9009  * @param[in] item
9010  *   Flow pattern to translate.
9011  * @param[in] pattern_flags
9012  *   Accumulated pattern flags.
9013  */
9014 static void
9015 flow_dv_translate_item_nvgre(void *matcher, void *key,
9016                              const struct rte_flow_item *item,
9017                              unsigned long pattern_flags)
9018 {
9019         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
9020         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
9021         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9022         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9023         const char *tni_flow_id_m;
9024         const char *tni_flow_id_v;
9025         char *gre_key_m;
9026         char *gre_key_v;
9027         int size;
9028         int i;
9029
9030         /* For NVGRE, GRE header fields must be set with defined values. */
9031         const struct rte_flow_item_gre gre_spec = {
9032                 .c_rsvd0_ver = RTE_BE16(0x2000),
9033                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
9034         };
9035         const struct rte_flow_item_gre gre_mask = {
9036                 .c_rsvd0_ver = RTE_BE16(0xB000),
9037                 .protocol = RTE_BE16(UINT16_MAX),
9038         };
9039         const struct rte_flow_item gre_item = {
9040                 .spec = &gre_spec,
9041                 .mask = &gre_mask,
9042                 .last = NULL,
9043         };
9044         flow_dv_translate_item_gre(matcher, key, &gre_item, pattern_flags);
9045         if (!nvgre_v)
9046                 return;
9047         if (!nvgre_m)
9048                 nvgre_m = &rte_flow_item_nvgre_mask;
9049         tni_flow_id_m = (const char *)nvgre_m->tni;
9050         tni_flow_id_v = (const char *)nvgre_v->tni;
9051         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
9052         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
9053         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
9054         memcpy(gre_key_m, tni_flow_id_m, size);
9055         for (i = 0; i < size; ++i)
9056                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
9057 }
9058
9059 /**
9060  * Add VXLAN item to matcher and to the value.
9061  *
9062  * @param[in] dev
9063  *   Pointer to the Ethernet device structure.
9064  * @param[in] attr
9065  *   Flow rule attributes.
9066  * @param[in, out] matcher
9067  *   Flow matcher.
9068  * @param[in, out] key
9069  *   Flow matcher value.
9070  * @param[in] item
9071  *   Flow pattern to translate.
9072  * @param[in] inner
9073  *   Item is inner pattern.
9074  */
9075 static void
9076 flow_dv_translate_item_vxlan(struct rte_eth_dev *dev,
9077                              const struct rte_flow_attr *attr,
9078                              void *matcher, void *key,
9079                              const struct rte_flow_item *item,
9080                              int inner)
9081 {
9082         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
9083         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
9084         void *headers_m;
9085         void *headers_v;
9086         void *misc5_m;
9087         void *misc5_v;
9088         uint32_t *tunnel_header_v;
9089         uint32_t *tunnel_header_m;
9090         uint16_t dport;
9091         struct mlx5_priv *priv = dev->data->dev_private;
9092         const struct rte_flow_item_vxlan nic_mask = {
9093                 .vni = "\xff\xff\xff",
9094                 .rsvd1 = 0xff,
9095         };
9096
9097         if (inner) {
9098                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9099                                          inner_headers);
9100                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9101         } else {
9102                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9103                                          outer_headers);
9104                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9105         }
9106         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
9107                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
9108         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9109                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9110                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9111         }
9112         dport = MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport);
9113         if (!vxlan_v)
9114                 return;
9115         if (!vxlan_m) {
9116                 if ((!attr->group && !priv->sh->tunnel_header_0_1) ||
9117                     (attr->group && !priv->sh->misc5_cap))
9118                         vxlan_m = &rte_flow_item_vxlan_mask;
9119                 else
9120                         vxlan_m = &nic_mask;
9121         }
9122         if ((priv->sh->steering_format_version ==
9123             MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 &&
9124             dport != MLX5_UDP_PORT_VXLAN) ||
9125             (!attr->group && !attr->transfer && !priv->sh->tunnel_header_0_1) ||
9126             ((attr->group || attr->transfer) && !priv->sh->misc5_cap)) {
9127                 void *misc_m;
9128                 void *misc_v;
9129                 char *vni_m;
9130                 char *vni_v;
9131                 int size;
9132                 int i;
9133                 misc_m = MLX5_ADDR_OF(fte_match_param,
9134                                       matcher, misc_parameters);
9135                 misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9136                 size = sizeof(vxlan_m->vni);
9137                 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
9138                 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
9139                 memcpy(vni_m, vxlan_m->vni, size);
9140                 for (i = 0; i < size; ++i)
9141                         vni_v[i] = vni_m[i] & vxlan_v->vni[i];
9142                 return;
9143         }
9144         misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
9145         misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
9146         tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
9147                                                    misc5_v,
9148                                                    tunnel_header_1);
9149         tunnel_header_m = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
9150                                                    misc5_m,
9151                                                    tunnel_header_1);
9152         *tunnel_header_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) |
9153                            (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 |
9154                            (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16;
9155         if (*tunnel_header_v)
9156                 *tunnel_header_m = vxlan_m->vni[0] |
9157                         vxlan_m->vni[1] << 8 |
9158                         vxlan_m->vni[2] << 16;
9159         else
9160                 *tunnel_header_m = 0x0;
9161         *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24;
9162         if (vxlan_v->rsvd1 & vxlan_m->rsvd1)
9163                 *tunnel_header_m |= vxlan_m->rsvd1 << 24;
9164 }
9165
9166 /**
9167  * Add VXLAN-GPE item to matcher and to the value.
9168  *
9169  * @param[in, out] matcher
9170  *   Flow matcher.
9171  * @param[in, out] key
9172  *   Flow matcher value.
9173  * @param[in] item
9174  *   Flow pattern to translate.
9175  * @param[in] inner
9176  *   Item is inner pattern.
9177  */
9178
9179 static void
9180 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
9181                                  const struct rte_flow_item *item,
9182                                  const uint64_t pattern_flags)
9183 {
9184         static const struct rte_flow_item_vxlan_gpe dummy_vxlan_gpe_hdr = {0, };
9185         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
9186         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
9187         /* The item was validated to be on the outer side */
9188         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9189         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9190         void *misc_m =
9191                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
9192         void *misc_v =
9193                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9194         char *vni_m =
9195                 MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
9196         char *vni_v =
9197                 MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
9198         int i, size = sizeof(vxlan_m->vni);
9199         uint8_t flags_m = 0xff;
9200         uint8_t flags_v = 0xc;
9201         uint8_t m_protocol, v_protocol;
9202
9203         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9204                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9205                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9206                          MLX5_UDP_PORT_VXLAN_GPE);
9207         }
9208         if (!vxlan_v) {
9209                 vxlan_v = &dummy_vxlan_gpe_hdr;
9210                 vxlan_m = &dummy_vxlan_gpe_hdr;
9211         } else {
9212                 if (!vxlan_m)
9213                         vxlan_m = &rte_flow_item_vxlan_gpe_mask;
9214         }
9215         memcpy(vni_m, vxlan_m->vni, size);
9216         for (i = 0; i < size; ++i)
9217                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
9218         if (vxlan_m->flags) {
9219                 flags_m = vxlan_m->flags;
9220                 flags_v = vxlan_v->flags;
9221         }
9222         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
9223         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
9224         m_protocol = vxlan_m->protocol;
9225         v_protocol = vxlan_v->protocol;
9226         if (!m_protocol) {
9227                 /* Force next protocol to ensure next headers parsing. */
9228                 if (pattern_flags & MLX5_FLOW_LAYER_INNER_L2)
9229                         v_protocol = RTE_VXLAN_GPE_TYPE_ETH;
9230                 else if (pattern_flags & MLX5_FLOW_LAYER_INNER_L3_IPV4)
9231                         v_protocol = RTE_VXLAN_GPE_TYPE_IPV4;
9232                 else if (pattern_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)
9233                         v_protocol = RTE_VXLAN_GPE_TYPE_IPV6;
9234                 if (v_protocol)
9235                         m_protocol = 0xFF;
9236         }
9237         MLX5_SET(fte_match_set_misc3, misc_m,
9238                  outer_vxlan_gpe_next_protocol, m_protocol);
9239         MLX5_SET(fte_match_set_misc3, misc_v,
9240                  outer_vxlan_gpe_next_protocol, m_protocol & v_protocol);
9241 }
9242
9243 /**
9244  * Add Geneve item to matcher and to the value.
9245  *
9246  * @param[in, out] matcher
9247  *   Flow matcher.
9248  * @param[in, out] key
9249  *   Flow matcher value.
9250  * @param[in] item
9251  *   Flow pattern to translate.
9252  * @param[in] inner
9253  *   Item is inner pattern.
9254  */
9255
9256 static void
9257 flow_dv_translate_item_geneve(void *matcher, void *key,
9258                               const struct rte_flow_item *item,
9259                               uint64_t pattern_flags)
9260 {
9261         static const struct rte_flow_item_geneve empty_geneve = {0,};
9262         const struct rte_flow_item_geneve *geneve_m = item->mask;
9263         const struct rte_flow_item_geneve *geneve_v = item->spec;
9264         /* GENEVE flow item validation allows single tunnel item */
9265         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9266         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9267         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9268         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9269         uint16_t gbhdr_m;
9270         uint16_t gbhdr_v;
9271         char *vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
9272         char *vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
9273         size_t size = sizeof(geneve_m->vni), i;
9274         uint16_t protocol_m, protocol_v;
9275
9276         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9277                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9278                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9279                          MLX5_UDP_PORT_GENEVE);
9280         }
9281         if (!geneve_v) {
9282                 geneve_v = &empty_geneve;
9283                 geneve_m = &empty_geneve;
9284         } else {
9285                 if (!geneve_m)
9286                         geneve_m = &rte_flow_item_geneve_mask;
9287         }
9288         memcpy(vni_m, geneve_m->vni, size);
9289         for (i = 0; i < size; ++i)
9290                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
9291         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
9292         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
9293         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
9294                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9295         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
9296                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9297         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9298                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9299         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9300                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
9301                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9302         protocol_m = rte_be_to_cpu_16(geneve_m->protocol);
9303         protocol_v = rte_be_to_cpu_16(geneve_v->protocol);
9304         if (!protocol_m) {
9305                 /* Force next protocol to prevent matchers duplication */
9306                 protocol_v = mlx5_translate_tunnel_etypes(pattern_flags);
9307                 if (protocol_v)
9308                         protocol_m = 0xFFFF;
9309         }
9310         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type, protocol_m);
9311         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
9312                  protocol_m & protocol_v);
9313 }
9314
9315 /**
9316  * Create Geneve TLV option resource.
9317  *
9318  * @param dev[in, out]
9319  *   Pointer to rte_eth_dev structure.
9320  * @param[in, out] tag_be24
9321  *   Tag value in big endian then R-shift 8.
9322  * @parm[in, out] dev_flow
9323  *   Pointer to the dev_flow.
9324  * @param[out] error
9325  *   pointer to error structure.
9326  *
9327  * @return
9328  *   0 on success otherwise -errno and errno is set.
9329  */
9330
9331 int
9332 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
9333                                              const struct rte_flow_item *item,
9334                                              struct rte_flow_error *error)
9335 {
9336         struct mlx5_priv *priv = dev->data->dev_private;
9337         struct mlx5_dev_ctx_shared *sh = priv->sh;
9338         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
9339                         sh->geneve_tlv_option_resource;
9340         struct mlx5_devx_obj *obj;
9341         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9342         int ret = 0;
9343
9344         if (!geneve_opt_v)
9345                 return -1;
9346         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
9347         if (geneve_opt_resource != NULL) {
9348                 if (geneve_opt_resource->option_class ==
9349                         geneve_opt_v->option_class &&
9350                         geneve_opt_resource->option_type ==
9351                         geneve_opt_v->option_type &&
9352                         geneve_opt_resource->length ==
9353                         geneve_opt_v->option_len) {
9354                         /* We already have GENEVE TLV option obj allocated. */
9355                         __atomic_fetch_add(&geneve_opt_resource->refcnt, 1,
9356                                            __ATOMIC_RELAXED);
9357                 } else {
9358                         ret = rte_flow_error_set(error, ENOMEM,
9359                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9360                                 "Only one GENEVE TLV option supported");
9361                         goto exit;
9362                 }
9363         } else {
9364                 /* Create a GENEVE TLV object and resource. */
9365                 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->cdev->ctx,
9366                                 geneve_opt_v->option_class,
9367                                 geneve_opt_v->option_type,
9368                                 geneve_opt_v->option_len);
9369                 if (!obj) {
9370                         ret = rte_flow_error_set(error, ENODATA,
9371                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9372                                 "Failed to create GENEVE TLV Devx object");
9373                         goto exit;
9374                 }
9375                 sh->geneve_tlv_option_resource =
9376                                 mlx5_malloc(MLX5_MEM_ZERO,
9377                                                 sizeof(*geneve_opt_resource),
9378                                                 0, SOCKET_ID_ANY);
9379                 if (!sh->geneve_tlv_option_resource) {
9380                         claim_zero(mlx5_devx_cmd_destroy(obj));
9381                         ret = rte_flow_error_set(error, ENOMEM,
9382                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9383                                 "GENEVE TLV object memory allocation failed");
9384                         goto exit;
9385                 }
9386                 geneve_opt_resource = sh->geneve_tlv_option_resource;
9387                 geneve_opt_resource->obj = obj;
9388                 geneve_opt_resource->option_class = geneve_opt_v->option_class;
9389                 geneve_opt_resource->option_type = geneve_opt_v->option_type;
9390                 geneve_opt_resource->length = geneve_opt_v->option_len;
9391                 __atomic_store_n(&geneve_opt_resource->refcnt, 1,
9392                                 __ATOMIC_RELAXED);
9393         }
9394 exit:
9395         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
9396         return ret;
9397 }
9398
9399 /**
9400  * Add Geneve TLV option item to matcher.
9401  *
9402  * @param[in, out] dev
9403  *   Pointer to rte_eth_dev structure.
9404  * @param[in, out] matcher
9405  *   Flow matcher.
9406  * @param[in, out] key
9407  *   Flow matcher value.
9408  * @param[in] item
9409  *   Flow pattern to translate.
9410  * @param[out] error
9411  *   Pointer to error structure.
9412  */
9413 static int
9414 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher,
9415                                   void *key, const struct rte_flow_item *item,
9416                                   struct rte_flow_error *error)
9417 {
9418         const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask;
9419         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9420         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9421         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9422         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9423                         misc_parameters_3);
9424         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9425         rte_be32_t opt_data_key = 0, opt_data_mask = 0;
9426         int ret = 0;
9427
9428         if (!geneve_opt_v)
9429                 return -1;
9430         if (!geneve_opt_m)
9431                 geneve_opt_m = &rte_flow_item_geneve_opt_mask;
9432         ret = flow_dev_geneve_tlv_option_resource_register(dev, item,
9433                                                            error);
9434         if (ret) {
9435                 DRV_LOG(ERR, "Failed to create geneve_tlv_obj");
9436                 return ret;
9437         }
9438         /*
9439          * Set the option length in GENEVE header if not requested.
9440          * The GENEVE TLV option length is expressed by the option length field
9441          * in the GENEVE header.
9442          * If the option length was not requested but the GENEVE TLV option item
9443          * is present we set the option length field implicitly.
9444          */
9445         if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) {
9446                 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9447                          MLX5_GENEVE_OPTLEN_MASK);
9448                 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9449                          geneve_opt_v->option_len + 1);
9450         }
9451         MLX5_SET(fte_match_set_misc, misc_m, geneve_tlv_option_0_exist, 1);
9452         MLX5_SET(fte_match_set_misc, misc_v, geneve_tlv_option_0_exist, 1);
9453         /* Set the data. */
9454         if (geneve_opt_v->data) {
9455                 memcpy(&opt_data_key, geneve_opt_v->data,
9456                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9457                                 sizeof(opt_data_key)));
9458                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9459                                 sizeof(opt_data_key));
9460                 memcpy(&opt_data_mask, geneve_opt_m->data,
9461                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9462                                 sizeof(opt_data_mask)));
9463                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9464                                 sizeof(opt_data_mask));
9465                 MLX5_SET(fte_match_set_misc3, misc3_m,
9466                                 geneve_tlv_option_0_data,
9467                                 rte_be_to_cpu_32(opt_data_mask));
9468                 MLX5_SET(fte_match_set_misc3, misc3_v,
9469                                 geneve_tlv_option_0_data,
9470                         rte_be_to_cpu_32(opt_data_key & opt_data_mask));
9471         }
9472         return ret;
9473 }
9474
9475 /**
9476  * Add MPLS item to matcher and to the value.
9477  *
9478  * @param[in, out] matcher
9479  *   Flow matcher.
9480  * @param[in, out] key
9481  *   Flow matcher value.
9482  * @param[in] item
9483  *   Flow pattern to translate.
9484  * @param[in] prev_layer
9485  *   The protocol layer indicated in previous item.
9486  * @param[in] inner
9487  *   Item is inner pattern.
9488  */
9489 static void
9490 flow_dv_translate_item_mpls(void *matcher, void *key,
9491                             const struct rte_flow_item *item,
9492                             uint64_t prev_layer,
9493                             int inner)
9494 {
9495         const uint32_t *in_mpls_m = item->mask;
9496         const uint32_t *in_mpls_v = item->spec;
9497         uint32_t *out_mpls_m = 0;
9498         uint32_t *out_mpls_v = 0;
9499         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9500         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9501         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
9502                                      misc_parameters_2);
9503         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9504         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9505         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9506
9507         switch (prev_layer) {
9508         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9509                 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9510                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
9511                                  0xffff);
9512                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9513                                  MLX5_UDP_PORT_MPLS);
9514                 }
9515                 break;
9516         case MLX5_FLOW_LAYER_GRE:
9517                 /* Fall-through. */
9518         case MLX5_FLOW_LAYER_GRE_KEY:
9519                 if (!MLX5_GET16(fte_match_set_misc, misc_v, gre_protocol)) {
9520                         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
9521                                  0xffff);
9522                         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
9523                                  RTE_ETHER_TYPE_MPLS);
9524                 }
9525                 break;
9526         default:
9527                 break;
9528         }
9529         if (!in_mpls_v)
9530                 return;
9531         if (!in_mpls_m)
9532                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
9533         switch (prev_layer) {
9534         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9535                 out_mpls_m =
9536                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9537                                                  outer_first_mpls_over_udp);
9538                 out_mpls_v =
9539                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9540                                                  outer_first_mpls_over_udp);
9541                 break;
9542         case MLX5_FLOW_LAYER_GRE:
9543                 out_mpls_m =
9544                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9545                                                  outer_first_mpls_over_gre);
9546                 out_mpls_v =
9547                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9548                                                  outer_first_mpls_over_gre);
9549                 break;
9550         default:
9551                 /* Inner MPLS not over GRE is not supported. */
9552                 if (!inner) {
9553                         out_mpls_m =
9554                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9555                                                          misc2_m,
9556                                                          outer_first_mpls);
9557                         out_mpls_v =
9558                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9559                                                          misc2_v,
9560                                                          outer_first_mpls);
9561                 }
9562                 break;
9563         }
9564         if (out_mpls_m && out_mpls_v) {
9565                 *out_mpls_m = *in_mpls_m;
9566                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
9567         }
9568 }
9569
9570 /**
9571  * Add metadata register item to matcher
9572  *
9573  * @param[in, out] matcher
9574  *   Flow matcher.
9575  * @param[in, out] key
9576  *   Flow matcher value.
9577  * @param[in] reg_type
9578  *   Type of device metadata register
9579  * @param[in] value
9580  *   Register value
9581  * @param[in] mask
9582  *   Register mask
9583  */
9584 static void
9585 flow_dv_match_meta_reg(void *matcher, void *key,
9586                        enum modify_reg reg_type,
9587                        uint32_t data, uint32_t mask)
9588 {
9589         void *misc2_m =
9590                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
9591         void *misc2_v =
9592                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9593         uint32_t temp;
9594
9595         data &= mask;
9596         switch (reg_type) {
9597         case REG_A:
9598                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
9599                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
9600                 break;
9601         case REG_B:
9602                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
9603                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
9604                 break;
9605         case REG_C_0:
9606                 /*
9607                  * The metadata register C0 field might be divided into
9608                  * source vport index and META item value, we should set
9609                  * this field according to specified mask, not as whole one.
9610                  */
9611                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
9612                 temp |= mask;
9613                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
9614                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
9615                 temp &= ~mask;
9616                 temp |= data;
9617                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
9618                 break;
9619         case REG_C_1:
9620                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
9621                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
9622                 break;
9623         case REG_C_2:
9624                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
9625                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
9626                 break;
9627         case REG_C_3:
9628                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
9629                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
9630                 break;
9631         case REG_C_4:
9632                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
9633                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
9634                 break;
9635         case REG_C_5:
9636                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
9637                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
9638                 break;
9639         case REG_C_6:
9640                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
9641                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
9642                 break;
9643         case REG_C_7:
9644                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
9645                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
9646                 break;
9647         default:
9648                 MLX5_ASSERT(false);
9649                 break;
9650         }
9651 }
9652
9653 /**
9654  * Add MARK item to matcher
9655  *
9656  * @param[in] dev
9657  *   The device to configure through.
9658  * @param[in, out] matcher
9659  *   Flow matcher.
9660  * @param[in, out] key
9661  *   Flow matcher value.
9662  * @param[in] item
9663  *   Flow pattern to translate.
9664  */
9665 static void
9666 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
9667                             void *matcher, void *key,
9668                             const struct rte_flow_item *item)
9669 {
9670         struct mlx5_priv *priv = dev->data->dev_private;
9671         const struct rte_flow_item_mark *mark;
9672         uint32_t value;
9673         uint32_t mask;
9674
9675         mark = item->mask ? (const void *)item->mask :
9676                             &rte_flow_item_mark_mask;
9677         mask = mark->id & priv->sh->dv_mark_mask;
9678         mark = (const void *)item->spec;
9679         MLX5_ASSERT(mark);
9680         value = mark->id & priv->sh->dv_mark_mask & mask;
9681         if (mask) {
9682                 enum modify_reg reg;
9683
9684                 /* Get the metadata register index for the mark. */
9685                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
9686                 MLX5_ASSERT(reg > 0);
9687                 if (reg == REG_C_0) {
9688                         struct mlx5_priv *priv = dev->data->dev_private;
9689                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9690                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9691
9692                         mask &= msk_c0;
9693                         mask <<= shl_c0;
9694                         value <<= shl_c0;
9695                 }
9696                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9697         }
9698 }
9699
9700 /**
9701  * Add META item to matcher
9702  *
9703  * @param[in] dev
9704  *   The devich to configure through.
9705  * @param[in, out] matcher
9706  *   Flow matcher.
9707  * @param[in, out] key
9708  *   Flow matcher value.
9709  * @param[in] attr
9710  *   Attributes of flow that includes this item.
9711  * @param[in] item
9712  *   Flow pattern to translate.
9713  */
9714 static void
9715 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
9716                             void *matcher, void *key,
9717                             const struct rte_flow_attr *attr,
9718                             const struct rte_flow_item *item)
9719 {
9720         const struct rte_flow_item_meta *meta_m;
9721         const struct rte_flow_item_meta *meta_v;
9722
9723         meta_m = (const void *)item->mask;
9724         if (!meta_m)
9725                 meta_m = &rte_flow_item_meta_mask;
9726         meta_v = (const void *)item->spec;
9727         if (meta_v) {
9728                 int reg;
9729                 uint32_t value = meta_v->data;
9730                 uint32_t mask = meta_m->data;
9731
9732                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
9733                 if (reg < 0)
9734                         return;
9735                 MLX5_ASSERT(reg != REG_NON);
9736                 if (reg == REG_C_0) {
9737                         struct mlx5_priv *priv = dev->data->dev_private;
9738                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9739                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9740
9741                         mask &= msk_c0;
9742                         mask <<= shl_c0;
9743                         value <<= shl_c0;
9744                 }
9745                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9746         }
9747 }
9748
9749 /**
9750  * Add vport metadata Reg C0 item to matcher
9751  *
9752  * @param[in, out] matcher
9753  *   Flow matcher.
9754  * @param[in, out] key
9755  *   Flow matcher value.
9756  * @param[in] reg
9757  *   Flow pattern to translate.
9758  */
9759 static void
9760 flow_dv_translate_item_meta_vport(void *matcher, void *key,
9761                                   uint32_t value, uint32_t mask)
9762 {
9763         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
9764 }
9765
9766 /**
9767  * Add tag item to matcher
9768  *
9769  * @param[in] dev
9770  *   The devich to configure through.
9771  * @param[in, out] matcher
9772  *   Flow matcher.
9773  * @param[in, out] key
9774  *   Flow matcher value.
9775  * @param[in] item
9776  *   Flow pattern to translate.
9777  */
9778 static void
9779 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
9780                                 void *matcher, void *key,
9781                                 const struct rte_flow_item *item)
9782 {
9783         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
9784         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
9785         uint32_t mask, value;
9786
9787         MLX5_ASSERT(tag_v);
9788         value = tag_v->data;
9789         mask = tag_m ? tag_m->data : UINT32_MAX;
9790         if (tag_v->id == REG_C_0) {
9791                 struct mlx5_priv *priv = dev->data->dev_private;
9792                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9793                 uint32_t shl_c0 = rte_bsf32(msk_c0);
9794
9795                 mask &= msk_c0;
9796                 mask <<= shl_c0;
9797                 value <<= shl_c0;
9798         }
9799         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
9800 }
9801
9802 /**
9803  * Add TAG item to matcher
9804  *
9805  * @param[in] dev
9806  *   The devich to configure through.
9807  * @param[in, out] matcher
9808  *   Flow matcher.
9809  * @param[in, out] key
9810  *   Flow matcher value.
9811  * @param[in] item
9812  *   Flow pattern to translate.
9813  */
9814 static void
9815 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
9816                            void *matcher, void *key,
9817                            const struct rte_flow_item *item)
9818 {
9819         const struct rte_flow_item_tag *tag_v = item->spec;
9820         const struct rte_flow_item_tag *tag_m = item->mask;
9821         enum modify_reg reg;
9822
9823         MLX5_ASSERT(tag_v);
9824         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
9825         /* Get the metadata register index for the tag. */
9826         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
9827         MLX5_ASSERT(reg > 0);
9828         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
9829 }
9830
9831 /**
9832  * Add source vport match to the specified matcher.
9833  *
9834  * @param[in, out] matcher
9835  *   Flow matcher.
9836  * @param[in, out] key
9837  *   Flow matcher value.
9838  * @param[in] port
9839  *   Source vport value to match
9840  * @param[in] mask
9841  *   Mask
9842  */
9843 static void
9844 flow_dv_translate_item_source_vport(void *matcher, void *key,
9845                                     int16_t port, uint16_t mask)
9846 {
9847         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9848         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9849
9850         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
9851         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
9852 }
9853
9854 /**
9855  * Translate port-id item to eswitch match on  port-id.
9856  *
9857  * @param[in] dev
9858  *   The devich to configure through.
9859  * @param[in, out] matcher
9860  *   Flow matcher.
9861  * @param[in, out] key
9862  *   Flow matcher value.
9863  * @param[in] item
9864  *   Flow pattern to translate.
9865  * @param[in]
9866  *   Flow attributes.
9867  *
9868  * @return
9869  *   0 on success, a negative errno value otherwise.
9870  */
9871 static int
9872 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
9873                                void *key, const struct rte_flow_item *item,
9874                                const struct rte_flow_attr *attr)
9875 {
9876         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
9877         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
9878         struct mlx5_priv *priv;
9879         uint16_t mask, id;
9880
9881         if (pid_v && pid_v->id == MLX5_PORT_ESW_MGR) {
9882                 flow_dv_translate_item_source_vport(matcher, key,
9883                         flow_dv_get_esw_manager_vport_id(dev), 0xffff);
9884                 return 0;
9885         }
9886         mask = pid_m ? pid_m->id : 0xffff;
9887         id = pid_v ? pid_v->id : dev->data->port_id;
9888         priv = mlx5_port_to_eswitch_info(id, item == NULL);
9889         if (!priv)
9890                 return -rte_errno;
9891         /*
9892          * Translate to vport field or to metadata, depending on mode.
9893          * Kernel can use either misc.source_port or half of C0 metadata
9894          * register.
9895          */
9896         if (priv->vport_meta_mask) {
9897                 /*
9898                  * Provide the hint for SW steering library
9899                  * to insert the flow into ingress domain and
9900                  * save the extra vport match.
9901                  */
9902                 if (mask == 0xffff && priv->vport_id == 0xffff &&
9903                     priv->pf_bond < 0 && attr->transfer)
9904                         flow_dv_translate_item_source_vport
9905                                 (matcher, key, priv->vport_id, mask);
9906                 /*
9907                  * We should always set the vport metadata register,
9908                  * otherwise the SW steering library can drop
9909                  * the rule if wire vport metadata value is not zero,
9910                  * it depends on kernel configuration.
9911                  */
9912                 flow_dv_translate_item_meta_vport(matcher, key,
9913                                                   priv->vport_meta_tag,
9914                                                   priv->vport_meta_mask);
9915         } else {
9916                 flow_dv_translate_item_source_vport(matcher, key,
9917                                                     priv->vport_id, mask);
9918         }
9919         return 0;
9920 }
9921
9922 /**
9923  * Add ICMP6 item to matcher and to the value.
9924  *
9925  * @param[in, out] matcher
9926  *   Flow matcher.
9927  * @param[in, out] key
9928  *   Flow matcher value.
9929  * @param[in] item
9930  *   Flow pattern to translate.
9931  * @param[in] inner
9932  *   Item is inner pattern.
9933  */
9934 static void
9935 flow_dv_translate_item_icmp6(void *matcher, void *key,
9936                               const struct rte_flow_item *item,
9937                               int inner)
9938 {
9939         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
9940         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
9941         void *headers_m;
9942         void *headers_v;
9943         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9944                                      misc_parameters_3);
9945         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9946         if (inner) {
9947                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9948                                          inner_headers);
9949                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9950         } else {
9951                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9952                                          outer_headers);
9953                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9954         }
9955         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9956         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
9957         if (!icmp6_v)
9958                 return;
9959         if (!icmp6_m)
9960                 icmp6_m = &rte_flow_item_icmp6_mask;
9961         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
9962         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
9963                  icmp6_v->type & icmp6_m->type);
9964         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
9965         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
9966                  icmp6_v->code & icmp6_m->code);
9967 }
9968
9969 /**
9970  * Add ICMP item to matcher and to the value.
9971  *
9972  * @param[in, out] matcher
9973  *   Flow matcher.
9974  * @param[in, out] key
9975  *   Flow matcher value.
9976  * @param[in] item
9977  *   Flow pattern to translate.
9978  * @param[in] inner
9979  *   Item is inner pattern.
9980  */
9981 static void
9982 flow_dv_translate_item_icmp(void *matcher, void *key,
9983                             const struct rte_flow_item *item,
9984                             int inner)
9985 {
9986         const struct rte_flow_item_icmp *icmp_m = item->mask;
9987         const struct rte_flow_item_icmp *icmp_v = item->spec;
9988         uint32_t icmp_header_data_m = 0;
9989         uint32_t icmp_header_data_v = 0;
9990         void *headers_m;
9991         void *headers_v;
9992         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9993                                      misc_parameters_3);
9994         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9995         if (inner) {
9996                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9997                                          inner_headers);
9998                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9999         } else {
10000                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
10001                                          outer_headers);
10002                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
10003         }
10004         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
10005         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
10006         if (!icmp_v)
10007                 return;
10008         if (!icmp_m)
10009                 icmp_m = &rte_flow_item_icmp_mask;
10010         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
10011                  icmp_m->hdr.icmp_type);
10012         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
10013                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
10014         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
10015                  icmp_m->hdr.icmp_code);
10016         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
10017                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
10018         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
10019         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
10020         if (icmp_header_data_m) {
10021                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
10022                 icmp_header_data_v |=
10023                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
10024                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
10025                          icmp_header_data_m);
10026                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
10027                          icmp_header_data_v & icmp_header_data_m);
10028         }
10029 }
10030
10031 /**
10032  * Add GTP item to matcher and to the value.
10033  *
10034  * @param[in, out] matcher
10035  *   Flow matcher.
10036  * @param[in, out] key
10037  *   Flow matcher value.
10038  * @param[in] item
10039  *   Flow pattern to translate.
10040  * @param[in] inner
10041  *   Item is inner pattern.
10042  */
10043 static void
10044 flow_dv_translate_item_gtp(void *matcher, void *key,
10045                            const struct rte_flow_item *item, int inner)
10046 {
10047         const struct rte_flow_item_gtp *gtp_m = item->mask;
10048         const struct rte_flow_item_gtp *gtp_v = item->spec;
10049         void *headers_m;
10050         void *headers_v;
10051         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
10052                                      misc_parameters_3);
10053         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
10054         uint16_t dport = RTE_GTPU_UDP_PORT;
10055
10056         if (inner) {
10057                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
10058                                          inner_headers);
10059                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
10060         } else {
10061                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
10062                                          outer_headers);
10063                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
10064         }
10065         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
10066                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
10067                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
10068         }
10069         if (!gtp_v)
10070                 return;
10071         if (!gtp_m)
10072                 gtp_m = &rte_flow_item_gtp_mask;
10073         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
10074                  gtp_m->v_pt_rsv_flags);
10075         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
10076                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
10077         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
10078         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
10079                  gtp_v->msg_type & gtp_m->msg_type);
10080         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
10081                  rte_be_to_cpu_32(gtp_m->teid));
10082         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
10083                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
10084 }
10085
10086 /**
10087  * Add GTP PSC item to matcher.
10088  *
10089  * @param[in, out] matcher
10090  *   Flow matcher.
10091  * @param[in, out] key
10092  *   Flow matcher value.
10093  * @param[in] item
10094  *   Flow pattern to translate.
10095  */
10096 static int
10097 flow_dv_translate_item_gtp_psc(void *matcher, void *key,
10098                                const struct rte_flow_item *item)
10099 {
10100         const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask;
10101         const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec;
10102         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
10103                         misc_parameters_3);
10104         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
10105         union {
10106                 uint32_t w32;
10107                 struct {
10108                         uint16_t seq_num;
10109                         uint8_t npdu_num;
10110                         uint8_t next_ext_header_type;
10111                 };
10112         } dw_2;
10113         uint8_t gtp_flags;
10114
10115         /* Always set E-flag match on one, regardless of GTP item settings. */
10116         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags);
10117         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
10118         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags);
10119         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags);
10120         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
10121         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags);
10122         /*Set next extension header type. */
10123         dw_2.seq_num = 0;
10124         dw_2.npdu_num = 0;
10125         dw_2.next_ext_header_type = 0xff;
10126         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2,
10127                  rte_cpu_to_be_32(dw_2.w32));
10128         dw_2.seq_num = 0;
10129         dw_2.npdu_num = 0;
10130         dw_2.next_ext_header_type = 0x85;
10131         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2,
10132                  rte_cpu_to_be_32(dw_2.w32));
10133         if (gtp_psc_v) {
10134                 union {
10135                         uint32_t w32;
10136                         struct {
10137                                 uint8_t len;
10138                                 uint8_t type_flags;
10139                                 uint8_t qfi;
10140                                 uint8_t reserved;
10141                         };
10142                 } dw_0;
10143
10144                 /*Set extension header PDU type and Qos. */
10145                 if (!gtp_psc_m)
10146                         gtp_psc_m = &rte_flow_item_gtp_psc_mask;
10147                 dw_0.w32 = 0;
10148                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->hdr.type);
10149                 dw_0.qfi = gtp_psc_m->hdr.qfi;
10150                 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0,
10151                          rte_cpu_to_be_32(dw_0.w32));
10152                 dw_0.w32 = 0;
10153                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->hdr.type &
10154                                                         gtp_psc_m->hdr.type);
10155                 dw_0.qfi = gtp_psc_v->hdr.qfi & gtp_psc_m->hdr.qfi;
10156                 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0,
10157                          rte_cpu_to_be_32(dw_0.w32));
10158         }
10159         return 0;
10160 }
10161
10162 /**
10163  * Add eCPRI item to matcher and to the value.
10164  *
10165  * @param[in] dev
10166  *   The devich to configure through.
10167  * @param[in, out] matcher
10168  *   Flow matcher.
10169  * @param[in, out] key
10170  *   Flow matcher value.
10171  * @param[in] item
10172  *   Flow pattern to translate.
10173  * @param[in] last_item
10174  *   Last item flags.
10175  */
10176 static void
10177 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
10178                              void *key, const struct rte_flow_item *item,
10179                              uint64_t last_item)
10180 {
10181         struct mlx5_priv *priv = dev->data->dev_private;
10182         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
10183         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
10184         struct rte_ecpri_common_hdr common;
10185         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
10186                                      misc_parameters_4);
10187         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
10188         uint32_t *samples;
10189         void *dw_m;
10190         void *dw_v;
10191
10192         /*
10193          * In case of eCPRI over Ethernet, if EtherType is not specified,
10194          * match on eCPRI EtherType implicitly.
10195          */
10196         if (last_item & MLX5_FLOW_LAYER_OUTER_L2) {
10197                 void *hdrs_m, *hdrs_v, *l2m, *l2v;
10198
10199                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
10200                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
10201                 l2m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, ethertype);
10202                 l2v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
10203                 if (*(uint16_t *)l2m == 0 && *(uint16_t *)l2v == 0) {
10204                         *(uint16_t *)l2m = UINT16_MAX;
10205                         *(uint16_t *)l2v = RTE_BE16(RTE_ETHER_TYPE_ECPRI);
10206                 }
10207         }
10208         if (!ecpri_v)
10209                 return;
10210         if (!ecpri_m)
10211                 ecpri_m = &rte_flow_item_ecpri_mask;
10212         /*
10213          * Maximal four DW samples are supported in a single matching now.
10214          * Two are used now for a eCPRI matching:
10215          * 1. Type: one byte, mask should be 0x00ff0000 in network order
10216          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
10217          *    if any.
10218          */
10219         if (!ecpri_m->hdr.common.u32)
10220                 return;
10221         samples = priv->sh->ecpri_parser.ids;
10222         /* Need to take the whole DW as the mask to fill the entry. */
10223         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
10224                             prog_sample_field_value_0);
10225         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
10226                             prog_sample_field_value_0);
10227         /* Already big endian (network order) in the header. */
10228         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
10229         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32;
10230         /* Sample#0, used for matching type, offset 0. */
10231         MLX5_SET(fte_match_set_misc4, misc4_m,
10232                  prog_sample_field_id_0, samples[0]);
10233         /* It makes no sense to set the sample ID in the mask field. */
10234         MLX5_SET(fte_match_set_misc4, misc4_v,
10235                  prog_sample_field_id_0, samples[0]);
10236         /*
10237          * Checking if message body part needs to be matched.
10238          * Some wildcard rules only matching type field should be supported.
10239          */
10240         if (ecpri_m->hdr.dummy[0]) {
10241                 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32);
10242                 switch (common.type) {
10243                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
10244                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
10245                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
10246                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
10247                                             prog_sample_field_value_1);
10248                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
10249                                             prog_sample_field_value_1);
10250                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
10251                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] &
10252                                             ecpri_m->hdr.dummy[0];
10253                         /* Sample#1, to match message body, offset 4. */
10254                         MLX5_SET(fte_match_set_misc4, misc4_m,
10255                                  prog_sample_field_id_1, samples[1]);
10256                         MLX5_SET(fte_match_set_misc4, misc4_v,
10257                                  prog_sample_field_id_1, samples[1]);
10258                         break;
10259                 default:
10260                         /* Others, do not match any sample ID. */
10261                         break;
10262                 }
10263         }
10264 }
10265
10266 /*
10267  * Add connection tracking status item to matcher
10268  *
10269  * @param[in] dev
10270  *   The devich to configure through.
10271  * @param[in, out] matcher
10272  *   Flow matcher.
10273  * @param[in, out] key
10274  *   Flow matcher value.
10275  * @param[in] item
10276  *   Flow pattern to translate.
10277  */
10278 static void
10279 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev,
10280                               void *matcher, void *key,
10281                               const struct rte_flow_item *item)
10282 {
10283         uint32_t reg_value = 0;
10284         int reg_id;
10285         /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */
10286         uint32_t reg_mask = 0;
10287         const struct rte_flow_item_conntrack *spec = item->spec;
10288         const struct rte_flow_item_conntrack *mask = item->mask;
10289         uint32_t flags;
10290         struct rte_flow_error error;
10291
10292         if (!mask)
10293                 mask = &rte_flow_item_conntrack_mask;
10294         if (!spec || !mask->flags)
10295                 return;
10296         flags = spec->flags & mask->flags;
10297         /* The conflict should be checked in the validation. */
10298         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID)
10299                 reg_value |= MLX5_CT_SYNDROME_VALID;
10300         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10301                 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE;
10302         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID)
10303                 reg_value |= MLX5_CT_SYNDROME_INVALID;
10304         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)
10305                 reg_value |= MLX5_CT_SYNDROME_TRAP;
10306         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10307                 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET;
10308         if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID |
10309                            RTE_FLOW_CONNTRACK_PKT_STATE_INVALID |
10310                            RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))
10311                 reg_mask |= 0xc0;
10312         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10313                 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE;
10314         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10315                 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET;
10316         /* The REG_C_x value could be saved during startup. */
10317         reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error);
10318         if (reg_id == REG_NON)
10319                 return;
10320         flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id,
10321                                reg_value, reg_mask);
10322 }
10323
10324 static void
10325 flow_dv_translate_item_flex(struct rte_eth_dev *dev, void *matcher, void *key,
10326                             const struct rte_flow_item *item,
10327                             struct mlx5_flow *dev_flow, bool is_inner)
10328 {
10329         const struct rte_flow_item_flex *spec =
10330                 (const struct rte_flow_item_flex *)item->spec;
10331         int index = mlx5_flex_acquire_index(dev, spec->handle, false);
10332
10333         MLX5_ASSERT(index >= 0 && index <= (int)(sizeof(uint32_t) * CHAR_BIT));
10334         if (index < 0)
10335                 return;
10336         if (!(dev_flow->handle->flex_item & RTE_BIT32(index))) {
10337                 /* Don't count both inner and outer flex items in one rule. */
10338                 if (mlx5_flex_acquire_index(dev, spec->handle, true) != index)
10339                         MLX5_ASSERT(false);
10340                 dev_flow->handle->flex_item |= (uint8_t)RTE_BIT32(index);
10341         }
10342         mlx5_flex_flow_translate_item(dev, matcher, key, item, is_inner);
10343 }
10344
10345 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
10346
10347 #define HEADER_IS_ZERO(match_criteria, headers)                              \
10348         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
10349                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
10350
10351 /**
10352  * Calculate flow matcher enable bitmap.
10353  *
10354  * @param match_criteria
10355  *   Pointer to flow matcher criteria.
10356  *
10357  * @return
10358  *   Bitmap of enabled fields.
10359  */
10360 static uint8_t
10361 flow_dv_matcher_enable(uint32_t *match_criteria)
10362 {
10363         uint8_t match_criteria_enable;
10364
10365         match_criteria_enable =
10366                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
10367                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
10368         match_criteria_enable |=
10369                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
10370                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
10371         match_criteria_enable |=
10372                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
10373                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
10374         match_criteria_enable |=
10375                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
10376                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
10377         match_criteria_enable |=
10378                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
10379                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
10380         match_criteria_enable |=
10381                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
10382                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
10383         match_criteria_enable |=
10384                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_5)) <<
10385                 MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT;
10386         return match_criteria_enable;
10387 }
10388
10389 static void
10390 __flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria)
10391 {
10392         /*
10393          * Check flow matching criteria first, subtract misc5/4 length if flow
10394          * doesn't own misc5/4 parameters. In some old rdma-core releases,
10395          * misc5/4 are not supported, and matcher creation failure is expected
10396          * w/o subtraction. If misc5 is provided, misc4 must be counted in since
10397          * misc5 is right after misc4.
10398          */
10399         if (!(match_criteria & (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT))) {
10400                 *size = MLX5_ST_SZ_BYTES(fte_match_param) -
10401                         MLX5_ST_SZ_BYTES(fte_match_set_misc5);
10402                 if (!(match_criteria & (1 <<
10403                         MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT))) {
10404                         *size -= MLX5_ST_SZ_BYTES(fte_match_set_misc4);
10405                 }
10406         }
10407 }
10408
10409 static struct mlx5_list_entry *
10410 flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
10411                          struct mlx5_list_entry *entry, void *cb_ctx)
10412 {
10413         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10414         struct mlx5_flow_dv_matcher *ref = ctx->data;
10415         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10416                                                             typeof(*tbl), tbl);
10417         struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
10418                                                             sizeof(*resource),
10419                                                             0, SOCKET_ID_ANY);
10420
10421         if (!resource) {
10422                 rte_flow_error_set(ctx->error, ENOMEM,
10423                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10424                                    "cannot create matcher");
10425                 return NULL;
10426         }
10427         memcpy(resource, entry, sizeof(*resource));
10428         resource->tbl = &tbl->tbl;
10429         return &resource->entry;
10430 }
10431
10432 static void
10433 flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
10434                              struct mlx5_list_entry *entry)
10435 {
10436         mlx5_free(entry);
10437 }
10438
10439 struct mlx5_list_entry *
10440 flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
10441 {
10442         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10443         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10444         struct rte_eth_dev *dev = ctx->dev;
10445         struct mlx5_flow_tbl_data_entry *tbl_data;
10446         struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
10447         struct rte_flow_error *error = ctx->error;
10448         union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
10449         struct mlx5_flow_tbl_resource *tbl;
10450         void *domain;
10451         uint32_t idx = 0;
10452         int ret;
10453
10454         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10455         if (!tbl_data) {
10456                 rte_flow_error_set(error, ENOMEM,
10457                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10458                                    NULL,
10459                                    "cannot allocate flow table data entry");
10460                 return NULL;
10461         }
10462         tbl_data->idx = idx;
10463         tbl_data->tunnel = tt_prm->tunnel;
10464         tbl_data->group_id = tt_prm->group_id;
10465         tbl_data->external = !!tt_prm->external;
10466         tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
10467         tbl_data->is_egress = !!key.is_egress;
10468         tbl_data->is_transfer = !!key.is_fdb;
10469         tbl_data->dummy = !!key.dummy;
10470         tbl_data->level = key.level;
10471         tbl_data->id = key.id;
10472         tbl = &tbl_data->tbl;
10473         if (key.dummy)
10474                 return &tbl_data->entry;
10475         if (key.is_fdb)
10476                 domain = sh->fdb_domain;
10477         else if (key.is_egress)
10478                 domain = sh->tx_domain;
10479         else
10480                 domain = sh->rx_domain;
10481         ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
10482         if (ret) {
10483                 rte_flow_error_set(error, ENOMEM,
10484                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10485                                    NULL, "cannot create flow table object");
10486                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10487                 return NULL;
10488         }
10489         if (key.level != 0) {
10490                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
10491                                         (tbl->obj, &tbl_data->jump.action);
10492                 if (ret) {
10493                         rte_flow_error_set(error, ENOMEM,
10494                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10495                                            NULL,
10496                                            "cannot create flow jump action");
10497                         mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10498                         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10499                         return NULL;
10500                 }
10501         }
10502         MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
10503               key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
10504               key.level, key.id);
10505         tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
10506                                               flow_dv_matcher_create_cb,
10507                                               flow_dv_matcher_match_cb,
10508                                               flow_dv_matcher_remove_cb,
10509                                               flow_dv_matcher_clone_cb,
10510                                               flow_dv_matcher_clone_free_cb);
10511         if (!tbl_data->matchers) {
10512                 rte_flow_error_set(error, ENOMEM,
10513                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10514                                    NULL,
10515                                    "cannot create tbl matcher list");
10516                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10517                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10518                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10519                 return NULL;
10520         }
10521         return &tbl_data->entry;
10522 }
10523
10524 int
10525 flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10526                      void *cb_ctx)
10527 {
10528         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10529         struct mlx5_flow_tbl_data_entry *tbl_data =
10530                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10531         union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
10532
10533         return tbl_data->level != key.level ||
10534                tbl_data->id != key.id ||
10535                tbl_data->dummy != key.dummy ||
10536                tbl_data->is_transfer != !!key.is_fdb ||
10537                tbl_data->is_egress != !!key.is_egress;
10538 }
10539
10540 struct mlx5_list_entry *
10541 flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10542                       void *cb_ctx)
10543 {
10544         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10545         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10546         struct mlx5_flow_tbl_data_entry *tbl_data;
10547         struct rte_flow_error *error = ctx->error;
10548         uint32_t idx = 0;
10549
10550         tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10551         if (!tbl_data) {
10552                 rte_flow_error_set(error, ENOMEM,
10553                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10554                                    NULL,
10555                                    "cannot allocate flow table data entry");
10556                 return NULL;
10557         }
10558         memcpy(tbl_data, oentry, sizeof(*tbl_data));
10559         tbl_data->idx = idx;
10560         return &tbl_data->entry;
10561 }
10562
10563 void
10564 flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10565 {
10566         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10567         struct mlx5_flow_tbl_data_entry *tbl_data =
10568                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10569
10570         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10571 }
10572
10573 /**
10574  * Get a flow table.
10575  *
10576  * @param[in, out] dev
10577  *   Pointer to rte_eth_dev structure.
10578  * @param[in] table_level
10579  *   Table level to use.
10580  * @param[in] egress
10581  *   Direction of the table.
10582  * @param[in] transfer
10583  *   E-Switch or NIC flow.
10584  * @param[in] dummy
10585  *   Dummy entry for dv API.
10586  * @param[in] table_id
10587  *   Table id to use.
10588  * @param[out] error
10589  *   pointer to error structure.
10590  *
10591  * @return
10592  *   Returns tables resource based on the index, NULL in case of failed.
10593  */
10594 struct mlx5_flow_tbl_resource *
10595 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
10596                          uint32_t table_level, uint8_t egress,
10597                          uint8_t transfer,
10598                          bool external,
10599                          const struct mlx5_flow_tunnel *tunnel,
10600                          uint32_t group_id, uint8_t dummy,
10601                          uint32_t table_id,
10602                          struct rte_flow_error *error)
10603 {
10604         struct mlx5_priv *priv = dev->data->dev_private;
10605         union mlx5_flow_tbl_key table_key = {
10606                 {
10607                         .level = table_level,
10608                         .id = table_id,
10609                         .reserved = 0,
10610                         .dummy = !!dummy,
10611                         .is_fdb = !!transfer,
10612                         .is_egress = !!egress,
10613                 }
10614         };
10615         struct mlx5_flow_tbl_tunnel_prm tt_prm = {
10616                 .tunnel = tunnel,
10617                 .group_id = group_id,
10618                 .external = external,
10619         };
10620         struct mlx5_flow_cb_ctx ctx = {
10621                 .dev = dev,
10622                 .error = error,
10623                 .data = &table_key.v64,
10624                 .data2 = &tt_prm,
10625         };
10626         struct mlx5_list_entry *entry;
10627         struct mlx5_flow_tbl_data_entry *tbl_data;
10628
10629         entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
10630         if (!entry) {
10631                 rte_flow_error_set(error, ENOMEM,
10632                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10633                                    "cannot get table");
10634                 return NULL;
10635         }
10636         DRV_LOG(DEBUG, "table_level %u table_id %u "
10637                 "tunnel %u group %u registered.",
10638                 table_level, table_id,
10639                 tunnel ? tunnel->tunnel_id : 0, group_id);
10640         tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10641         return &tbl_data->tbl;
10642 }
10643
10644 void
10645 flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10646 {
10647         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10648         struct mlx5_flow_tbl_data_entry *tbl_data =
10649                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10650
10651         MLX5_ASSERT(entry && sh);
10652         if (tbl_data->jump.action)
10653                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10654         if (tbl_data->tbl.obj)
10655                 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
10656         if (tbl_data->tunnel_offload && tbl_data->external) {
10657                 struct mlx5_list_entry *he;
10658                 struct mlx5_hlist *tunnel_grp_hash;
10659                 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10660                 union tunnel_tbl_key tunnel_key = {
10661                         .tunnel_id = tbl_data->tunnel ?
10662                                         tbl_data->tunnel->tunnel_id : 0,
10663                         .group = tbl_data->group_id
10664                 };
10665                 uint32_t table_level = tbl_data->level;
10666                 struct mlx5_flow_cb_ctx ctx = {
10667                         .data = (void *)&tunnel_key.val,
10668                 };
10669
10670                 tunnel_grp_hash = tbl_data->tunnel ?
10671                                         tbl_data->tunnel->groups :
10672                                         thub->groups;
10673                 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
10674                 if (he)
10675                         mlx5_hlist_unregister(tunnel_grp_hash, he);
10676                 DRV_LOG(DEBUG,
10677                         "table_level %u id %u tunnel %u group %u released.",
10678                         table_level,
10679                         tbl_data->id,
10680                         tbl_data->tunnel ?
10681                         tbl_data->tunnel->tunnel_id : 0,
10682                         tbl_data->group_id);
10683         }
10684         if (tbl_data->matchers)
10685                 mlx5_list_destroy(tbl_data->matchers);
10686         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10687 }
10688
10689 /**
10690  * Release a flow table.
10691  *
10692  * @param[in] sh
10693  *   Pointer to device shared structure.
10694  * @param[in] tbl
10695  *   Table resource to be released.
10696  *
10697  * @return
10698  *   Returns 0 if table was released, else return 1;
10699  */
10700 static int
10701 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
10702                              struct mlx5_flow_tbl_resource *tbl)
10703 {
10704         struct mlx5_flow_tbl_data_entry *tbl_data =
10705                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10706
10707         if (!tbl)
10708                 return 0;
10709         return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
10710 }
10711
10712 int
10713 flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
10714                          struct mlx5_list_entry *entry, void *cb_ctx)
10715 {
10716         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10717         struct mlx5_flow_dv_matcher *ref = ctx->data;
10718         struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur),
10719                                                         entry);
10720
10721         return cur->crc != ref->crc ||
10722                cur->priority != ref->priority ||
10723                memcmp((const void *)cur->mask.buf,
10724                       (const void *)ref->mask.buf, ref->mask.size);
10725 }
10726
10727 struct mlx5_list_entry *
10728 flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
10729 {
10730         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10731         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10732         struct mlx5_flow_dv_matcher *ref = ctx->data;
10733         struct mlx5_flow_dv_matcher *resource;
10734         struct mlx5dv_flow_matcher_attr dv_attr = {
10735                 .type = IBV_FLOW_ATTR_NORMAL,
10736                 .match_mask = (void *)&ref->mask,
10737         };
10738         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10739                                                             typeof(*tbl), tbl);
10740         int ret;
10741
10742         resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
10743                                SOCKET_ID_ANY);
10744         if (!resource) {
10745                 rte_flow_error_set(ctx->error, ENOMEM,
10746                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10747                                    "cannot create matcher");
10748                 return NULL;
10749         }
10750         *resource = *ref;
10751         dv_attr.match_criteria_enable =
10752                 flow_dv_matcher_enable(resource->mask.buf);
10753         __flow_dv_adjust_buf_size(&ref->mask.size,
10754                                   dv_attr.match_criteria_enable);
10755         dv_attr.priority = ref->priority;
10756         if (tbl->is_egress)
10757                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
10758         ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
10759                                                tbl->tbl.obj,
10760                                                &resource->matcher_object);
10761         if (ret) {
10762                 mlx5_free(resource);
10763                 rte_flow_error_set(ctx->error, ENOMEM,
10764                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10765                                    "cannot create matcher");
10766                 return NULL;
10767         }
10768         return &resource->entry;
10769 }
10770
10771 /**
10772  * Register the flow matcher.
10773  *
10774  * @param[in, out] dev
10775  *   Pointer to rte_eth_dev structure.
10776  * @param[in, out] matcher
10777  *   Pointer to flow matcher.
10778  * @param[in, out] key
10779  *   Pointer to flow table key.
10780  * @parm[in, out] dev_flow
10781  *   Pointer to the dev_flow.
10782  * @param[out] error
10783  *   pointer to error structure.
10784  *
10785  * @return
10786  *   0 on success otherwise -errno and errno is set.
10787  */
10788 static int
10789 flow_dv_matcher_register(struct rte_eth_dev *dev,
10790                          struct mlx5_flow_dv_matcher *ref,
10791                          union mlx5_flow_tbl_key *key,
10792                          struct mlx5_flow *dev_flow,
10793                          const struct mlx5_flow_tunnel *tunnel,
10794                          uint32_t group_id,
10795                          struct rte_flow_error *error)
10796 {
10797         struct mlx5_list_entry *entry;
10798         struct mlx5_flow_dv_matcher *resource;
10799         struct mlx5_flow_tbl_resource *tbl;
10800         struct mlx5_flow_tbl_data_entry *tbl_data;
10801         struct mlx5_flow_cb_ctx ctx = {
10802                 .error = error,
10803                 .data = ref,
10804         };
10805         /**
10806          * tunnel offload API requires this registration for cases when
10807          * tunnel match rule was inserted before tunnel set rule.
10808          */
10809         tbl = flow_dv_tbl_resource_get(dev, key->level,
10810                                        key->is_egress, key->is_fdb,
10811                                        dev_flow->external, tunnel,
10812                                        group_id, 0, key->id, error);
10813         if (!tbl)
10814                 return -rte_errno;      /* No need to refill the error info */
10815         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10816         ref->tbl = tbl;
10817         entry = mlx5_list_register(tbl_data->matchers, &ctx);
10818         if (!entry) {
10819                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
10820                 return rte_flow_error_set(error, ENOMEM,
10821                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10822                                           "cannot allocate ref memory");
10823         }
10824         resource = container_of(entry, typeof(*resource), entry);
10825         dev_flow->handle->dvh.matcher = resource;
10826         return 0;
10827 }
10828
10829 struct mlx5_list_entry *
10830 flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
10831 {
10832         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10833         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10834         struct mlx5_flow_dv_tag_resource *entry;
10835         uint32_t idx = 0;
10836         int ret;
10837
10838         entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10839         if (!entry) {
10840                 rte_flow_error_set(ctx->error, ENOMEM,
10841                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10842                                    "cannot allocate resource memory");
10843                 return NULL;
10844         }
10845         entry->idx = idx;
10846         entry->tag_id = *(uint32_t *)(ctx->data);
10847         ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
10848                                                   &entry->action);
10849         if (ret) {
10850                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
10851                 rte_flow_error_set(ctx->error, ENOMEM,
10852                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10853                                    NULL, "cannot create action");
10854                 return NULL;
10855         }
10856         return &entry->entry;
10857 }
10858
10859 int
10860 flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10861                      void *cb_ctx)
10862 {
10863         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10864         struct mlx5_flow_dv_tag_resource *tag =
10865                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10866
10867         return *(uint32_t *)(ctx->data) != tag->tag_id;
10868 }
10869
10870 struct mlx5_list_entry *
10871 flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10872                      void *cb_ctx)
10873 {
10874         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10875         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10876         struct mlx5_flow_dv_tag_resource *entry;
10877         uint32_t idx = 0;
10878
10879         entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10880         if (!entry) {
10881                 rte_flow_error_set(ctx->error, ENOMEM,
10882                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10883                                    "cannot allocate tag resource memory");
10884                 return NULL;
10885         }
10886         memcpy(entry, oentry, sizeof(*entry));
10887         entry->idx = idx;
10888         return &entry->entry;
10889 }
10890
10891 void
10892 flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10893 {
10894         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10895         struct mlx5_flow_dv_tag_resource *tag =
10896                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10897
10898         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10899 }
10900
10901 /**
10902  * Find existing tag resource or create and register a new one.
10903  *
10904  * @param dev[in, out]
10905  *   Pointer to rte_eth_dev structure.
10906  * @param[in, out] tag_be24
10907  *   Tag value in big endian then R-shift 8.
10908  * @parm[in, out] dev_flow
10909  *   Pointer to the dev_flow.
10910  * @param[out] error
10911  *   pointer to error structure.
10912  *
10913  * @return
10914  *   0 on success otherwise -errno and errno is set.
10915  */
10916 static int
10917 flow_dv_tag_resource_register
10918                         (struct rte_eth_dev *dev,
10919                          uint32_t tag_be24,
10920                          struct mlx5_flow *dev_flow,
10921                          struct rte_flow_error *error)
10922 {
10923         struct mlx5_priv *priv = dev->data->dev_private;
10924         struct mlx5_flow_dv_tag_resource *resource;
10925         struct mlx5_list_entry *entry;
10926         struct mlx5_flow_cb_ctx ctx = {
10927                                         .error = error,
10928                                         .data = &tag_be24,
10929                                         };
10930         struct mlx5_hlist *tag_table;
10931
10932         tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
10933                                       "tags",
10934                                       MLX5_TAGS_HLIST_ARRAY_SIZE,
10935                                       false, false, priv->sh,
10936                                       flow_dv_tag_create_cb,
10937                                       flow_dv_tag_match_cb,
10938                                       flow_dv_tag_remove_cb,
10939                                       flow_dv_tag_clone_cb,
10940                                       flow_dv_tag_clone_free_cb,
10941                                       error);
10942         if (unlikely(!tag_table))
10943                 return -rte_errno;
10944         entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
10945         if (entry) {
10946                 resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
10947                                         entry);
10948                 dev_flow->handle->dvh.rix_tag = resource->idx;
10949                 dev_flow->dv.tag_resource = resource;
10950                 return 0;
10951         }
10952         return -rte_errno;
10953 }
10954
10955 void
10956 flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10957 {
10958         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10959         struct mlx5_flow_dv_tag_resource *tag =
10960                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10961
10962         MLX5_ASSERT(tag && sh && tag->action);
10963         claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
10964         DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag);
10965         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10966 }
10967
10968 /**
10969  * Release the tag.
10970  *
10971  * @param dev
10972  *   Pointer to Ethernet device.
10973  * @param tag_idx
10974  *   Tag index.
10975  *
10976  * @return
10977  *   1 while a reference on it exists, 0 when freed.
10978  */
10979 static int
10980 flow_dv_tag_release(struct rte_eth_dev *dev,
10981                     uint32_t tag_idx)
10982 {
10983         struct mlx5_priv *priv = dev->data->dev_private;
10984         struct mlx5_flow_dv_tag_resource *tag;
10985
10986         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
10987         if (!tag)
10988                 return 0;
10989         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
10990                 dev->data->port_id, (void *)tag, tag->entry.ref_cnt);
10991         return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry);
10992 }
10993
10994 /**
10995  * Translate action PORT_ID / REPRESENTED_PORT to vport.
10996  *
10997  * @param[in] dev
10998  *   Pointer to rte_eth_dev structure.
10999  * @param[in] action
11000  *   Pointer to action PORT_ID / REPRESENTED_PORT.
11001  * @param[out] dst_port_id
11002  *   The target port ID.
11003  * @param[out] error
11004  *   Pointer to the error structure.
11005  *
11006  * @return
11007  *   0 on success, a negative errno value otherwise and rte_errno is set.
11008  */
11009 static int
11010 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
11011                                  const struct rte_flow_action *action,
11012                                  uint32_t *dst_port_id,
11013                                  struct rte_flow_error *error)
11014 {
11015         uint32_t port;
11016         struct mlx5_priv *priv;
11017
11018         switch (action->type) {
11019         case RTE_FLOW_ACTION_TYPE_PORT_ID: {
11020                 const struct rte_flow_action_port_id *conf;
11021
11022                 conf = (const struct rte_flow_action_port_id *)action->conf;
11023                 port = conf->original ? dev->data->port_id : conf->id;
11024                 break;
11025         }
11026         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
11027                 const struct rte_flow_action_ethdev *ethdev;
11028
11029                 ethdev = (const struct rte_flow_action_ethdev *)action->conf;
11030                 port = ethdev->port_id;
11031                 break;
11032         }
11033         default:
11034                 MLX5_ASSERT(false);
11035                 return rte_flow_error_set(error, EINVAL,
11036                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
11037                                           "unknown E-Switch action");
11038         }
11039
11040         priv = mlx5_port_to_eswitch_info(port, false);
11041         if (!priv)
11042                 return rte_flow_error_set(error, -rte_errno,
11043                                           RTE_FLOW_ERROR_TYPE_ACTION,
11044                                           NULL,
11045                                           "No eswitch info was found for port");
11046 #ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT
11047         /*
11048          * This parameter is transferred to
11049          * mlx5dv_dr_action_create_dest_ib_port().
11050          */
11051         *dst_port_id = priv->dev_port;
11052 #else
11053         /*
11054          * Legacy mode, no LAG configurations is supported.
11055          * This parameter is transferred to
11056          * mlx5dv_dr_action_create_dest_vport().
11057          */
11058         *dst_port_id = priv->vport_id;
11059 #endif
11060         return 0;
11061 }
11062
11063 /**
11064  * Create a counter with aging configuration.
11065  *
11066  * @param[in] dev
11067  *   Pointer to rte_eth_dev structure.
11068  * @param[in] dev_flow
11069  *   Pointer to the mlx5_flow.
11070  * @param[out] count
11071  *   Pointer to the counter action configuration.
11072  * @param[in] age
11073  *   Pointer to the aging action configuration.
11074  *
11075  * @return
11076  *   Index to flow counter on success, 0 otherwise.
11077  */
11078 static uint32_t
11079 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
11080                                 struct mlx5_flow *dev_flow,
11081                                 const struct rte_flow_action_count *count
11082                                         __rte_unused,
11083                                 const struct rte_flow_action_age *age)
11084 {
11085         uint32_t counter;
11086         struct mlx5_age_param *age_param;
11087
11088         counter = flow_dv_counter_alloc(dev, !!age);
11089         if (!counter || age == NULL)
11090                 return counter;
11091         age_param = flow_dv_counter_idx_get_age(dev, counter);
11092         age_param->context = age->context ? age->context :
11093                 (void *)(uintptr_t)(dev_flow->flow_idx);
11094         age_param->timeout = age->timeout;
11095         age_param->port_id = dev->data->port_id;
11096         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
11097         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
11098         return counter;
11099 }
11100
11101 /**
11102  * Add Tx queue matcher
11103  *
11104  * @param[in] dev
11105  *   Pointer to the dev struct.
11106  * @param[in, out] matcher
11107  *   Flow matcher.
11108  * @param[in, out] key
11109  *   Flow matcher value.
11110  * @param[in] item
11111  *   Flow pattern to translate.
11112  * @param[in] inner
11113  *   Item is inner pattern.
11114  */
11115 static void
11116 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
11117                                 void *matcher, void *key,
11118                                 const struct rte_flow_item *item)
11119 {
11120         const struct mlx5_rte_flow_item_tx_queue *queue_m;
11121         const struct mlx5_rte_flow_item_tx_queue *queue_v;
11122         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
11123         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
11124         struct mlx5_txq_ctrl *txq;
11125         uint32_t queue, mask;
11126
11127         queue_m = (const void *)item->mask;
11128         queue_v = (const void *)item->spec;
11129         if (!queue_v)
11130                 return;
11131         txq = mlx5_txq_get(dev, queue_v->queue);
11132         if (!txq)
11133                 return;
11134         if (txq->is_hairpin)
11135                 queue = txq->obj->sq->id;
11136         else
11137                 queue = txq->obj->sq_obj.sq->id;
11138         mask = queue_m == NULL ? UINT32_MAX : queue_m->queue;
11139         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, mask);
11140         MLX5_SET(fte_match_set_misc, misc_v, source_sqn, queue & mask);
11141         mlx5_txq_release(dev, queue_v->queue);
11142 }
11143
11144 /**
11145  * Set the hash fields according to the @p flow information.
11146  *
11147  * @param[in] item_flags
11148  *   The match pattern item flags.
11149  * @param[in] rss_desc
11150  *   Pointer to the mlx5_flow_rss_desc.
11151  * @param[out] hash_fields
11152  *   Pointer to the RSS hash fields.
11153  */
11154 void
11155 flow_dv_hashfields_set(uint64_t item_flags,
11156                        struct mlx5_flow_rss_desc *rss_desc,
11157                        uint64_t *hash_fields)
11158 {
11159         uint64_t items = item_flags;
11160         uint64_t fields = 0;
11161         int rss_inner = 0;
11162         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
11163
11164         *hash_fields = 0;
11165 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
11166         if (rss_desc->level >= 2)
11167                 rss_inner = 1;
11168 #endif
11169         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
11170             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4)) ||
11171              !items) {
11172                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
11173                         if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
11174                                 fields |= IBV_RX_HASH_SRC_IPV4;
11175                         else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
11176                                 fields |= IBV_RX_HASH_DST_IPV4;
11177                         else
11178                                 fields |= MLX5_IPV4_IBV_RX_HASH;
11179                 }
11180         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
11181                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6)) ||
11182                    !items) {
11183                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
11184                         if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
11185                                 fields |= IBV_RX_HASH_SRC_IPV6;
11186                         else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
11187                                 fields |= IBV_RX_HASH_DST_IPV6;
11188                         else
11189                                 fields |= MLX5_IPV6_IBV_RX_HASH;
11190                 }
11191         }
11192         if (fields == 0)
11193                 /*
11194                  * There is no match between the RSS types and the
11195                  * L3 protocol (IPv4/IPv6) defined in the flow rule.
11196                  */
11197                 return;
11198         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
11199             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP)) ||
11200             !items) {
11201                 if (rss_types & RTE_ETH_RSS_UDP) {
11202                         if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
11203                                 fields |= IBV_RX_HASH_SRC_PORT_UDP;
11204                         else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
11205                                 fields |= IBV_RX_HASH_DST_PORT_UDP;
11206                         else
11207                                 fields |= MLX5_UDP_IBV_RX_HASH;
11208                 }
11209         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
11210                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP)) ||
11211                    !items) {
11212                 if (rss_types & RTE_ETH_RSS_TCP) {
11213                         if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
11214                                 fields |= IBV_RX_HASH_SRC_PORT_TCP;
11215                         else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
11216                                 fields |= IBV_RX_HASH_DST_PORT_TCP;
11217                         else
11218                                 fields |= MLX5_TCP_IBV_RX_HASH;
11219                 }
11220         }
11221         if (rss_inner)
11222                 fields |= IBV_RX_HASH_INNER;
11223         *hash_fields = fields;
11224 }
11225
11226 /**
11227  * Prepare an Rx Hash queue.
11228  *
11229  * @param dev
11230  *   Pointer to Ethernet device.
11231  * @param[in] dev_flow
11232  *   Pointer to the mlx5_flow.
11233  * @param[in] rss_desc
11234  *   Pointer to the mlx5_flow_rss_desc.
11235  * @param[out] hrxq_idx
11236  *   Hash Rx queue index.
11237  *
11238  * @return
11239  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
11240  */
11241 static struct mlx5_hrxq *
11242 flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
11243                      struct mlx5_flow *dev_flow,
11244                      struct mlx5_flow_rss_desc *rss_desc,
11245                      uint32_t *hrxq_idx)
11246 {
11247         struct mlx5_flow_handle *dh = dev_flow->handle;
11248         uint32_t shared_rss = rss_desc->shared_rss;
11249         struct mlx5_hrxq *hrxq;
11250
11251         MLX5_ASSERT(rss_desc->queue_num);
11252         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
11253         rss_desc->hash_fields = dev_flow->hash_fields;
11254         rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL);
11255         rss_desc->shared_rss = 0;
11256         if (rss_desc->hash_fields == 0)
11257                 rss_desc->queue_num = 1;
11258         hrxq = mlx5_hrxq_get(dev, rss_desc);
11259         *hrxq_idx = hrxq ? hrxq->idx : 0;
11260         rss_desc->shared_rss = shared_rss;
11261         return hrxq;
11262 }
11263
11264 /**
11265  * Release sample sub action resource.
11266  *
11267  * @param[in, out] dev
11268  *   Pointer to rte_eth_dev structure.
11269  * @param[in] act_res
11270  *   Pointer to sample sub action resource.
11271  */
11272 static void
11273 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
11274                                    struct mlx5_flow_sub_actions_idx *act_res)
11275 {
11276         if (act_res->rix_hrxq) {
11277                 mlx5_hrxq_release(dev, act_res->rix_hrxq);
11278                 act_res->rix_hrxq = 0;
11279         }
11280         if (act_res->rix_encap_decap) {
11281                 flow_dv_encap_decap_resource_release(dev,
11282                                                      act_res->rix_encap_decap);
11283                 act_res->rix_encap_decap = 0;
11284         }
11285         if (act_res->rix_port_id_action) {
11286                 flow_dv_port_id_action_resource_release(dev,
11287                                                 act_res->rix_port_id_action);
11288                 act_res->rix_port_id_action = 0;
11289         }
11290         if (act_res->rix_tag) {
11291                 flow_dv_tag_release(dev, act_res->rix_tag);
11292                 act_res->rix_tag = 0;
11293         }
11294         if (act_res->rix_jump) {
11295                 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump);
11296                 act_res->rix_jump = 0;
11297         }
11298 }
11299
11300 int
11301 flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
11302                         struct mlx5_list_entry *entry, void *cb_ctx)
11303 {
11304         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11305         struct rte_eth_dev *dev = ctx->dev;
11306         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11307         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
11308                                                               typeof(*resource),
11309                                                               entry);
11310
11311         if (ctx_resource->ratio == resource->ratio &&
11312             ctx_resource->ft_type == resource->ft_type &&
11313             ctx_resource->ft_id == resource->ft_id &&
11314             ctx_resource->set_action == resource->set_action &&
11315             !memcmp((void *)&ctx_resource->sample_act,
11316                     (void *)&resource->sample_act,
11317                     sizeof(struct mlx5_flow_sub_actions_list))) {
11318                 /*
11319                  * Existing sample action should release the prepared
11320                  * sub-actions reference counter.
11321                  */
11322                 flow_dv_sample_sub_actions_release(dev,
11323                                                    &ctx_resource->sample_idx);
11324                 return 0;
11325         }
11326         return 1;
11327 }
11328
11329 struct mlx5_list_entry *
11330 flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11331 {
11332         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11333         struct rte_eth_dev *dev = ctx->dev;
11334         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11335         void **sample_dv_actions = ctx_resource->sub_actions;
11336         struct mlx5_flow_dv_sample_resource *resource;
11337         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
11338         struct mlx5_priv *priv = dev->data->dev_private;
11339         struct mlx5_dev_ctx_shared *sh = priv->sh;
11340         struct mlx5_flow_tbl_resource *tbl;
11341         uint32_t idx = 0;
11342         const uint32_t next_ft_step = 1;
11343         uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
11344         uint8_t is_egress = 0;
11345         uint8_t is_transfer = 0;
11346         struct rte_flow_error *error = ctx->error;
11347
11348         /* Register new sample resource. */
11349         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11350         if (!resource) {
11351                 rte_flow_error_set(error, ENOMEM,
11352                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11353                                           NULL,
11354                                           "cannot allocate resource memory");
11355                 return NULL;
11356         }
11357         *resource = *ctx_resource;
11358         /* Create normal path table level */
11359         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11360                 is_transfer = 1;
11361         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
11362                 is_egress = 1;
11363         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
11364                                         is_egress, is_transfer,
11365                                         true, NULL, 0, 0, 0, error);
11366         if (!tbl) {
11367                 rte_flow_error_set(error, ENOMEM,
11368                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11369                                           NULL,
11370                                           "fail to create normal path table "
11371                                           "for sample");
11372                 goto error;
11373         }
11374         resource->normal_path_tbl = tbl;
11375         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
11376                 if (!sh->default_miss_action) {
11377                         rte_flow_error_set(error, ENOMEM,
11378                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11379                                                 NULL,
11380                                                 "default miss action was not "
11381                                                 "created");
11382                         goto error;
11383                 }
11384                 sample_dv_actions[ctx_resource->sample_act.actions_num++] =
11385                                                 sh->default_miss_action;
11386         }
11387         /* Create a DR sample action */
11388         sampler_attr.sample_ratio = resource->ratio;
11389         sampler_attr.default_next_table = tbl->obj;
11390         sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
11391         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
11392                                                         &sample_dv_actions[0];
11393         sampler_attr.action = resource->set_action;
11394         if (mlx5_os_flow_dr_create_flow_action_sampler
11395                         (&sampler_attr, &resource->verbs_action)) {
11396                 rte_flow_error_set(error, ENOMEM,
11397                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11398                                         NULL, "cannot create sample action");
11399                 goto error;
11400         }
11401         resource->idx = idx;
11402         resource->dev = dev;
11403         return &resource->entry;
11404 error:
11405         if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
11406                 flow_dv_sample_sub_actions_release(dev,
11407                                                    &resource->sample_idx);
11408         if (resource->normal_path_tbl)
11409                 flow_dv_tbl_resource_release(MLX5_SH(dev),
11410                                 resource->normal_path_tbl);
11411         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
11412         return NULL;
11413
11414 }
11415
11416 struct mlx5_list_entry *
11417 flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
11418                          struct mlx5_list_entry *entry __rte_unused,
11419                          void *cb_ctx)
11420 {
11421         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11422         struct rte_eth_dev *dev = ctx->dev;
11423         struct mlx5_flow_dv_sample_resource *resource;
11424         struct mlx5_priv *priv = dev->data->dev_private;
11425         struct mlx5_dev_ctx_shared *sh = priv->sh;
11426         uint32_t idx = 0;
11427
11428         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11429         if (!resource) {
11430                 rte_flow_error_set(ctx->error, ENOMEM,
11431                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11432                                           NULL,
11433                                           "cannot allocate resource memory");
11434                 return NULL;
11435         }
11436         memcpy(resource, entry, sizeof(*resource));
11437         resource->idx = idx;
11438         resource->dev = dev;
11439         return &resource->entry;
11440 }
11441
11442 void
11443 flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
11444                              struct mlx5_list_entry *entry)
11445 {
11446         struct mlx5_flow_dv_sample_resource *resource =
11447                                   container_of(entry, typeof(*resource), entry);
11448         struct rte_eth_dev *dev = resource->dev;
11449         struct mlx5_priv *priv = dev->data->dev_private;
11450
11451         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
11452 }
11453
11454 /**
11455  * Find existing sample resource or create and register a new one.
11456  *
11457  * @param[in, out] dev
11458  *   Pointer to rte_eth_dev structure.
11459  * @param[in] ref
11460  *   Pointer to sample resource reference.
11461  * @parm[in, out] dev_flow
11462  *   Pointer to the dev_flow.
11463  * @param[out] error
11464  *   pointer to error structure.
11465  *
11466  * @return
11467  *   0 on success otherwise -errno and errno is set.
11468  */
11469 static int
11470 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
11471                          struct mlx5_flow_dv_sample_resource *ref,
11472                          struct mlx5_flow *dev_flow,
11473                          struct rte_flow_error *error)
11474 {
11475         struct mlx5_flow_dv_sample_resource *resource;
11476         struct mlx5_list_entry *entry;
11477         struct mlx5_priv *priv = dev->data->dev_private;
11478         struct mlx5_flow_cb_ctx ctx = {
11479                 .dev = dev,
11480                 .error = error,
11481                 .data = ref,
11482         };
11483
11484         entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
11485         if (!entry)
11486                 return -rte_errno;
11487         resource = container_of(entry, typeof(*resource), entry);
11488         dev_flow->handle->dvh.rix_sample = resource->idx;
11489         dev_flow->dv.sample_res = resource;
11490         return 0;
11491 }
11492
11493 int
11494 flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
11495                             struct mlx5_list_entry *entry, void *cb_ctx)
11496 {
11497         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11498         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11499         struct rte_eth_dev *dev = ctx->dev;
11500         struct mlx5_flow_dv_dest_array_resource *resource =
11501                                   container_of(entry, typeof(*resource), entry);
11502         uint32_t idx = 0;
11503
11504         if (ctx_resource->num_of_dest == resource->num_of_dest &&
11505             ctx_resource->ft_type == resource->ft_type &&
11506             !memcmp((void *)resource->sample_act,
11507                     (void *)ctx_resource->sample_act,
11508                    (ctx_resource->num_of_dest *
11509                    sizeof(struct mlx5_flow_sub_actions_list)))) {
11510                 /*
11511                  * Existing sample action should release the prepared
11512                  * sub-actions reference counter.
11513                  */
11514                 for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11515                         flow_dv_sample_sub_actions_release(dev,
11516                                         &ctx_resource->sample_idx[idx]);
11517                 return 0;
11518         }
11519         return 1;
11520 }
11521
11522 struct mlx5_list_entry *
11523 flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11524 {
11525         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11526         struct rte_eth_dev *dev = ctx->dev;
11527         struct mlx5_flow_dv_dest_array_resource *resource;
11528         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11529         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
11530         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
11531         struct mlx5_priv *priv = dev->data->dev_private;
11532         struct mlx5_dev_ctx_shared *sh = priv->sh;
11533         struct mlx5_flow_sub_actions_list *sample_act;
11534         struct mlx5dv_dr_domain *domain;
11535         uint32_t idx = 0, res_idx = 0;
11536         struct rte_flow_error *error = ctx->error;
11537         uint64_t action_flags;
11538         int ret;
11539
11540         /* Register new destination array resource. */
11541         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11542                                             &res_idx);
11543         if (!resource) {
11544                 rte_flow_error_set(error, ENOMEM,
11545                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11546                                           NULL,
11547                                           "cannot allocate resource memory");
11548                 return NULL;
11549         }
11550         *resource = *ctx_resource;
11551         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11552                 domain = sh->fdb_domain;
11553         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
11554                 domain = sh->rx_domain;
11555         else
11556                 domain = sh->tx_domain;
11557         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11558                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
11559                                  mlx5_malloc(MLX5_MEM_ZERO,
11560                                  sizeof(struct mlx5dv_dr_action_dest_attr),
11561                                  0, SOCKET_ID_ANY);
11562                 if (!dest_attr[idx]) {
11563                         rte_flow_error_set(error, ENOMEM,
11564                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11565                                            NULL,
11566                                            "cannot allocate resource memory");
11567                         goto error;
11568                 }
11569                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
11570                 sample_act = &ctx_resource->sample_act[idx];
11571                 action_flags = sample_act->action_flags;
11572                 switch (action_flags) {
11573                 case MLX5_FLOW_ACTION_QUEUE:
11574                         dest_attr[idx]->dest = sample_act->dr_queue_action;
11575                         break;
11576                 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP):
11577                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
11578                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
11579                         dest_attr[idx]->dest_reformat->reformat =
11580                                         sample_act->dr_encap_action;
11581                         dest_attr[idx]->dest_reformat->dest =
11582                                         sample_act->dr_port_id_action;
11583                         break;
11584                 case MLX5_FLOW_ACTION_PORT_ID:
11585                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
11586                         break;
11587                 case MLX5_FLOW_ACTION_JUMP:
11588                         dest_attr[idx]->dest = sample_act->dr_jump_action;
11589                         break;
11590                 default:
11591                         rte_flow_error_set(error, EINVAL,
11592                                            RTE_FLOW_ERROR_TYPE_ACTION,
11593                                            NULL,
11594                                            "unsupported actions type");
11595                         goto error;
11596                 }
11597         }
11598         /* create a dest array action */
11599         ret = mlx5_os_flow_dr_create_flow_action_dest_array
11600                                                 (domain,
11601                                                  resource->num_of_dest,
11602                                                  dest_attr,
11603                                                  &resource->action);
11604         if (ret) {
11605                 rte_flow_error_set(error, ENOMEM,
11606                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11607                                    NULL,
11608                                    "cannot create destination array action");
11609                 goto error;
11610         }
11611         resource->idx = res_idx;
11612         resource->dev = dev;
11613         for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11614                 mlx5_free(dest_attr[idx]);
11615         return &resource->entry;
11616 error:
11617         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11618                 flow_dv_sample_sub_actions_release(dev,
11619                                                    &resource->sample_idx[idx]);
11620                 if (dest_attr[idx])
11621                         mlx5_free(dest_attr[idx]);
11622         }
11623         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
11624         return NULL;
11625 }
11626
11627 struct mlx5_list_entry *
11628 flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
11629                             struct mlx5_list_entry *entry __rte_unused,
11630                             void *cb_ctx)
11631 {
11632         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11633         struct rte_eth_dev *dev = ctx->dev;
11634         struct mlx5_flow_dv_dest_array_resource *resource;
11635         struct mlx5_priv *priv = dev->data->dev_private;
11636         struct mlx5_dev_ctx_shared *sh = priv->sh;
11637         uint32_t res_idx = 0;
11638         struct rte_flow_error *error = ctx->error;
11639
11640         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11641                                       &res_idx);
11642         if (!resource) {
11643                 rte_flow_error_set(error, ENOMEM,
11644                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11645                                           NULL,
11646                                           "cannot allocate dest-array memory");
11647                 return NULL;
11648         }
11649         memcpy(resource, entry, sizeof(*resource));
11650         resource->idx = res_idx;
11651         resource->dev = dev;
11652         return &resource->entry;
11653 }
11654
11655 void
11656 flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
11657                                  struct mlx5_list_entry *entry)
11658 {
11659         struct mlx5_flow_dv_dest_array_resource *resource =
11660                         container_of(entry, typeof(*resource), entry);
11661         struct rte_eth_dev *dev = resource->dev;
11662         struct mlx5_priv *priv = dev->data->dev_private;
11663
11664         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
11665 }
11666
11667 /**
11668  * Find existing destination array resource or create and register a new one.
11669  *
11670  * @param[in, out] dev
11671  *   Pointer to rte_eth_dev structure.
11672  * @param[in] ref
11673  *   Pointer to destination array resource reference.
11674  * @parm[in, out] dev_flow
11675  *   Pointer to the dev_flow.
11676  * @param[out] error
11677  *   pointer to error structure.
11678  *
11679  * @return
11680  *   0 on success otherwise -errno and errno is set.
11681  */
11682 static int
11683 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
11684                          struct mlx5_flow_dv_dest_array_resource *ref,
11685                          struct mlx5_flow *dev_flow,
11686                          struct rte_flow_error *error)
11687 {
11688         struct mlx5_flow_dv_dest_array_resource *resource;
11689         struct mlx5_priv *priv = dev->data->dev_private;
11690         struct mlx5_list_entry *entry;
11691         struct mlx5_flow_cb_ctx ctx = {
11692                 .dev = dev,
11693                 .error = error,
11694                 .data = ref,
11695         };
11696
11697         entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
11698         if (!entry)
11699                 return -rte_errno;
11700         resource = container_of(entry, typeof(*resource), entry);
11701         dev_flow->handle->dvh.rix_dest_array = resource->idx;
11702         dev_flow->dv.dest_array_res = resource;
11703         return 0;
11704 }
11705
11706 /**
11707  * Convert Sample action to DV specification.
11708  *
11709  * @param[in] dev
11710  *   Pointer to rte_eth_dev structure.
11711  * @param[in] action
11712  *   Pointer to sample action structure.
11713  * @param[in, out] dev_flow
11714  *   Pointer to the mlx5_flow.
11715  * @param[in] attr
11716  *   Pointer to the flow attributes.
11717  * @param[in, out] num_of_dest
11718  *   Pointer to the num of destination.
11719  * @param[in, out] sample_actions
11720  *   Pointer to sample actions list.
11721  * @param[in, out] res
11722  *   Pointer to sample resource.
11723  * @param[out] error
11724  *   Pointer to the error structure.
11725  *
11726  * @return
11727  *   0 on success, a negative errno value otherwise and rte_errno is set.
11728  */
11729 static int
11730 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
11731                                 const struct rte_flow_action_sample *action,
11732                                 struct mlx5_flow *dev_flow,
11733                                 const struct rte_flow_attr *attr,
11734                                 uint32_t *num_of_dest,
11735                                 void **sample_actions,
11736                                 struct mlx5_flow_dv_sample_resource *res,
11737                                 struct rte_flow_error *error)
11738 {
11739         struct mlx5_priv *priv = dev->data->dev_private;
11740         const struct rte_flow_action *sub_actions;
11741         struct mlx5_flow_sub_actions_list *sample_act;
11742         struct mlx5_flow_sub_actions_idx *sample_idx;
11743         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11744         struct rte_flow *flow = dev_flow->flow;
11745         struct mlx5_flow_rss_desc *rss_desc;
11746         uint64_t action_flags = 0;
11747
11748         MLX5_ASSERT(wks);
11749         rss_desc = &wks->rss_desc;
11750         sample_act = &res->sample_act;
11751         sample_idx = &res->sample_idx;
11752         res->ratio = action->ratio;
11753         sub_actions = action->actions;
11754         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
11755                 int type = sub_actions->type;
11756                 uint32_t pre_rix = 0;
11757                 void *pre_r;
11758                 switch (type) {
11759                 case RTE_FLOW_ACTION_TYPE_QUEUE:
11760                 {
11761                         const struct rte_flow_action_queue *queue;
11762                         struct mlx5_hrxq *hrxq;
11763                         uint32_t hrxq_idx;
11764
11765                         queue = sub_actions->conf;
11766                         rss_desc->queue_num = 1;
11767                         rss_desc->queue[0] = queue->index;
11768                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11769                                                     rss_desc, &hrxq_idx);
11770                         if (!hrxq)
11771                                 return rte_flow_error_set
11772                                         (error, rte_errno,
11773                                          RTE_FLOW_ERROR_TYPE_ACTION,
11774                                          NULL,
11775                                          "cannot create fate queue");
11776                         sample_act->dr_queue_action = hrxq->action;
11777                         sample_idx->rix_hrxq = hrxq_idx;
11778                         sample_actions[sample_act->actions_num++] =
11779                                                 hrxq->action;
11780                         (*num_of_dest)++;
11781                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
11782                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11783                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11784                         dev_flow->handle->fate_action =
11785                                         MLX5_FLOW_FATE_QUEUE;
11786                         break;
11787                 }
11788                 case RTE_FLOW_ACTION_TYPE_RSS:
11789                 {
11790                         struct mlx5_hrxq *hrxq;
11791                         uint32_t hrxq_idx;
11792                         const struct rte_flow_action_rss *rss;
11793                         const uint8_t *rss_key;
11794
11795                         rss = sub_actions->conf;
11796                         memcpy(rss_desc->queue, rss->queue,
11797                                rss->queue_num * sizeof(uint16_t));
11798                         rss_desc->queue_num = rss->queue_num;
11799                         /* NULL RSS key indicates default RSS key. */
11800                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11801                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11802                         /*
11803                          * rss->level and rss.types should be set in advance
11804                          * when expanding items for RSS.
11805                          */
11806                         flow_dv_hashfields_set(dev_flow->handle->layers,
11807                                                rss_desc,
11808                                                &dev_flow->hash_fields);
11809                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11810                                                     rss_desc, &hrxq_idx);
11811                         if (!hrxq)
11812                                 return rte_flow_error_set
11813                                         (error, rte_errno,
11814                                          RTE_FLOW_ERROR_TYPE_ACTION,
11815                                          NULL,
11816                                          "cannot create fate queue");
11817                         sample_act->dr_queue_action = hrxq->action;
11818                         sample_idx->rix_hrxq = hrxq_idx;
11819                         sample_actions[sample_act->actions_num++] =
11820                                                 hrxq->action;
11821                         (*num_of_dest)++;
11822                         action_flags |= MLX5_FLOW_ACTION_RSS;
11823                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11824                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11825                         dev_flow->handle->fate_action =
11826                                         MLX5_FLOW_FATE_QUEUE;
11827                         break;
11828                 }
11829                 case RTE_FLOW_ACTION_TYPE_MARK:
11830                 {
11831                         uint32_t tag_be = mlx5_flow_mark_set
11832                                 (((const struct rte_flow_action_mark *)
11833                                 (sub_actions->conf))->id);
11834
11835                         wks->mark = 1;
11836                         pre_rix = dev_flow->handle->dvh.rix_tag;
11837                         /* Save the mark resource before sample */
11838                         pre_r = dev_flow->dv.tag_resource;
11839                         if (flow_dv_tag_resource_register(dev, tag_be,
11840                                                   dev_flow, error))
11841                                 return -rte_errno;
11842                         MLX5_ASSERT(dev_flow->dv.tag_resource);
11843                         sample_act->dr_tag_action =
11844                                 dev_flow->dv.tag_resource->action;
11845                         sample_idx->rix_tag =
11846                                 dev_flow->handle->dvh.rix_tag;
11847                         sample_actions[sample_act->actions_num++] =
11848                                                 sample_act->dr_tag_action;
11849                         /* Recover the mark resource after sample */
11850                         dev_flow->dv.tag_resource = pre_r;
11851                         dev_flow->handle->dvh.rix_tag = pre_rix;
11852                         action_flags |= MLX5_FLOW_ACTION_MARK;
11853                         break;
11854                 }
11855                 case RTE_FLOW_ACTION_TYPE_COUNT:
11856                 {
11857                         if (!flow->counter) {
11858                                 flow->counter =
11859                                         flow_dv_translate_create_counter(dev,
11860                                                 dev_flow, sub_actions->conf,
11861                                                 0);
11862                                 if (!flow->counter)
11863                                         return rte_flow_error_set
11864                                                 (error, rte_errno,
11865                                                 RTE_FLOW_ERROR_TYPE_ACTION,
11866                                                 NULL,
11867                                                 "cannot create counter"
11868                                                 " object.");
11869                         }
11870                         sample_act->dr_cnt_action =
11871                                   (flow_dv_counter_get_by_idx(dev,
11872                                   flow->counter, NULL))->action;
11873                         sample_actions[sample_act->actions_num++] =
11874                                                 sample_act->dr_cnt_action;
11875                         action_flags |= MLX5_FLOW_ACTION_COUNT;
11876                         break;
11877                 }
11878                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
11879                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
11880                 {
11881                         struct mlx5_flow_dv_port_id_action_resource
11882                                         port_id_resource;
11883                         uint32_t port_id = 0;
11884
11885                         memset(&port_id_resource, 0, sizeof(port_id_resource));
11886                         /* Save the port id resource before sample */
11887                         pre_rix = dev_flow->handle->rix_port_id_action;
11888                         pre_r = dev_flow->dv.port_id_action;
11889                         if (flow_dv_translate_action_port_id(dev, sub_actions,
11890                                                              &port_id, error))
11891                                 return -rte_errno;
11892                         port_id_resource.port_id = port_id;
11893                         if (flow_dv_port_id_action_resource_register
11894                             (dev, &port_id_resource, dev_flow, error))
11895                                 return -rte_errno;
11896                         sample_act->dr_port_id_action =
11897                                 dev_flow->dv.port_id_action->action;
11898                         sample_idx->rix_port_id_action =
11899                                 dev_flow->handle->rix_port_id_action;
11900                         sample_actions[sample_act->actions_num++] =
11901                                                 sample_act->dr_port_id_action;
11902                         /* Recover the port id resource after sample */
11903                         dev_flow->dv.port_id_action = pre_r;
11904                         dev_flow->handle->rix_port_id_action = pre_rix;
11905                         (*num_of_dest)++;
11906                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
11907                         break;
11908                 }
11909                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
11910                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
11911                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
11912                         /* Save the encap resource before sample */
11913                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
11914                         pre_r = dev_flow->dv.encap_decap;
11915                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
11916                                                            dev_flow,
11917                                                            attr->transfer,
11918                                                            error))
11919                                 return -rte_errno;
11920                         sample_act->dr_encap_action =
11921                                 dev_flow->dv.encap_decap->action;
11922                         sample_idx->rix_encap_decap =
11923                                 dev_flow->handle->dvh.rix_encap_decap;
11924                         sample_actions[sample_act->actions_num++] =
11925                                                 sample_act->dr_encap_action;
11926                         /* Recover the encap resource after sample */
11927                         dev_flow->dv.encap_decap = pre_r;
11928                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
11929                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
11930                         break;
11931                 default:
11932                         return rte_flow_error_set(error, EINVAL,
11933                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11934                                 NULL,
11935                                 "Not support for sampler action");
11936                 }
11937         }
11938         sample_act->action_flags = action_flags;
11939         res->ft_id = dev_flow->dv.group;
11940         if (attr->transfer) {
11941                 union {
11942                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
11943                         uint64_t set_action;
11944                 } action_ctx = { .set_action = 0 };
11945
11946                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
11947                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
11948                          MLX5_MODIFICATION_TYPE_SET);
11949                 MLX5_SET(set_action_in, action_ctx.action_in, field,
11950                          MLX5_MODI_META_REG_C_0);
11951                 MLX5_SET(set_action_in, action_ctx.action_in, data,
11952                          priv->vport_meta_tag);
11953                 res->set_action = action_ctx.set_action;
11954         } else if (attr->ingress) {
11955                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
11956         } else {
11957                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
11958         }
11959         return 0;
11960 }
11961
11962 /**
11963  * Convert Sample action to DV specification.
11964  *
11965  * @param[in] dev
11966  *   Pointer to rte_eth_dev structure.
11967  * @param[in, out] dev_flow
11968  *   Pointer to the mlx5_flow.
11969  * @param[in] num_of_dest
11970  *   The num of destination.
11971  * @param[in, out] res
11972  *   Pointer to sample resource.
11973  * @param[in, out] mdest_res
11974  *   Pointer to destination array resource.
11975  * @param[in] sample_actions
11976  *   Pointer to sample path actions list.
11977  * @param[in] action_flags
11978  *   Holds the actions detected until now.
11979  * @param[out] error
11980  *   Pointer to the error structure.
11981  *
11982  * @return
11983  *   0 on success, a negative errno value otherwise and rte_errno is set.
11984  */
11985 static int
11986 flow_dv_create_action_sample(struct rte_eth_dev *dev,
11987                              struct mlx5_flow *dev_flow,
11988                              uint32_t num_of_dest,
11989                              struct mlx5_flow_dv_sample_resource *res,
11990                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
11991                              void **sample_actions,
11992                              uint64_t action_flags,
11993                              struct rte_flow_error *error)
11994 {
11995         /* update normal path action resource into last index of array */
11996         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
11997         struct mlx5_flow_sub_actions_list *sample_act =
11998                                         &mdest_res->sample_act[dest_index];
11999         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12000         struct mlx5_flow_rss_desc *rss_desc;
12001         uint32_t normal_idx = 0;
12002         struct mlx5_hrxq *hrxq;
12003         uint32_t hrxq_idx;
12004
12005         MLX5_ASSERT(wks);
12006         rss_desc = &wks->rss_desc;
12007         if (num_of_dest > 1) {
12008                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
12009                         /* Handle QP action for mirroring */
12010                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
12011                                                     rss_desc, &hrxq_idx);
12012                         if (!hrxq)
12013                                 return rte_flow_error_set
12014                                      (error, rte_errno,
12015                                       RTE_FLOW_ERROR_TYPE_ACTION,
12016                                       NULL,
12017                                       "cannot create rx queue");
12018                         normal_idx++;
12019                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
12020                         sample_act->dr_queue_action = hrxq->action;
12021                         if (action_flags & MLX5_FLOW_ACTION_MARK)
12022                                 dev_flow->handle->rix_hrxq = hrxq_idx;
12023                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
12024                 }
12025                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
12026                         normal_idx++;
12027                         mdest_res->sample_idx[dest_index].rix_encap_decap =
12028                                 dev_flow->handle->dvh.rix_encap_decap;
12029                         sample_act->dr_encap_action =
12030                                 dev_flow->dv.encap_decap->action;
12031                         dev_flow->handle->dvh.rix_encap_decap = 0;
12032                 }
12033                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
12034                         normal_idx++;
12035                         mdest_res->sample_idx[dest_index].rix_port_id_action =
12036                                 dev_flow->handle->rix_port_id_action;
12037                         sample_act->dr_port_id_action =
12038                                 dev_flow->dv.port_id_action->action;
12039                         dev_flow->handle->rix_port_id_action = 0;
12040                 }
12041                 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) {
12042                         normal_idx++;
12043                         mdest_res->sample_idx[dest_index].rix_jump =
12044                                 dev_flow->handle->rix_jump;
12045                         sample_act->dr_jump_action =
12046                                 dev_flow->dv.jump->action;
12047                         dev_flow->handle->rix_jump = 0;
12048                 }
12049                 sample_act->actions_num = normal_idx;
12050                 /* update sample action resource into first index of array */
12051                 mdest_res->ft_type = res->ft_type;
12052                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
12053                                 sizeof(struct mlx5_flow_sub_actions_idx));
12054                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
12055                                 sizeof(struct mlx5_flow_sub_actions_list));
12056                 mdest_res->num_of_dest = num_of_dest;
12057                 if (flow_dv_dest_array_resource_register(dev, mdest_res,
12058                                                          dev_flow, error))
12059                         return rte_flow_error_set(error, EINVAL,
12060                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12061                                                   NULL, "can't create sample "
12062                                                   "action");
12063         } else {
12064                 res->sub_actions = sample_actions;
12065                 if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
12066                         return rte_flow_error_set(error, EINVAL,
12067                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12068                                                   NULL,
12069                                                   "can't create sample action");
12070         }
12071         return 0;
12072 }
12073
12074 /**
12075  * Remove an ASO age action from age actions list.
12076  *
12077  * @param[in] dev
12078  *   Pointer to the Ethernet device structure.
12079  * @param[in] age
12080  *   Pointer to the aso age action handler.
12081  */
12082 static void
12083 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev,
12084                                 struct mlx5_aso_age_action *age)
12085 {
12086         struct mlx5_age_info *age_info;
12087         struct mlx5_age_param *age_param = &age->age_params;
12088         struct mlx5_priv *priv = dev->data->dev_private;
12089         uint16_t expected = AGE_CANDIDATE;
12090
12091         age_info = GET_PORT_AGE_INFO(priv);
12092         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
12093                                          AGE_FREE, false, __ATOMIC_RELAXED,
12094                                          __ATOMIC_RELAXED)) {
12095                 /**
12096                  * We need the lock even it is age timeout,
12097                  * since age action may still in process.
12098                  */
12099                 rte_spinlock_lock(&age_info->aged_sl);
12100                 LIST_REMOVE(age, next);
12101                 rte_spinlock_unlock(&age_info->aged_sl);
12102                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
12103         }
12104 }
12105
12106 /**
12107  * Release an ASO age action.
12108  *
12109  * @param[in] dev
12110  *   Pointer to the Ethernet device structure.
12111  * @param[in] age_idx
12112  *   Index of ASO age action to release.
12113  * @param[in] flow
12114  *   True if the release operation is during flow destroy operation.
12115  *   False if the release operation is during action destroy operation.
12116  *
12117  * @return
12118  *   0 when age action was removed, otherwise the number of references.
12119  */
12120 static int
12121 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
12122 {
12123         struct mlx5_priv *priv = dev->data->dev_private;
12124         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12125         struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx);
12126         uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED);
12127
12128         if (!ret) {
12129                 flow_dv_aso_age_remove_from_age(dev, age);
12130                 rte_spinlock_lock(&mng->free_sl);
12131                 LIST_INSERT_HEAD(&mng->free, age, next);
12132                 rte_spinlock_unlock(&mng->free_sl);
12133         }
12134         return ret;
12135 }
12136
12137 /**
12138  * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
12139  *
12140  * @param[in] dev
12141  *   Pointer to the Ethernet device structure.
12142  *
12143  * @return
12144  *   0 on success, otherwise negative errno value and rte_errno is set.
12145  */
12146 static int
12147 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
12148 {
12149         struct mlx5_priv *priv = dev->data->dev_private;
12150         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12151         void *old_pools = mng->pools;
12152         uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
12153         uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
12154         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
12155
12156         if (!pools) {
12157                 rte_errno = ENOMEM;
12158                 return -ENOMEM;
12159         }
12160         if (old_pools) {
12161                 memcpy(pools, old_pools,
12162                        mng->n * sizeof(struct mlx5_flow_counter_pool *));
12163                 mlx5_free(old_pools);
12164         } else {
12165                 /* First ASO flow hit allocation - starting ASO data-path. */
12166                 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh);
12167
12168                 if (ret) {
12169                         mlx5_free(pools);
12170                         return ret;
12171                 }
12172         }
12173         mng->n = resize;
12174         mng->pools = pools;
12175         return 0;
12176 }
12177
12178 /**
12179  * Create and initialize a new ASO aging pool.
12180  *
12181  * @param[in] dev
12182  *   Pointer to the Ethernet device structure.
12183  * @param[out] age_free
12184  *   Where to put the pointer of a new age action.
12185  *
12186  * @return
12187  *   The age actions pool pointer and @p age_free is set on success,
12188  *   NULL otherwise and rte_errno is set.
12189  */
12190 static struct mlx5_aso_age_pool *
12191 flow_dv_age_pool_create(struct rte_eth_dev *dev,
12192                         struct mlx5_aso_age_action **age_free)
12193 {
12194         struct mlx5_priv *priv = dev->data->dev_private;
12195         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12196         struct mlx5_aso_age_pool *pool = NULL;
12197         struct mlx5_devx_obj *obj = NULL;
12198         uint32_t i;
12199
12200         obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->cdev->ctx,
12201                                                     priv->sh->cdev->pdn);
12202         if (!obj) {
12203                 rte_errno = ENODATA;
12204                 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX.");
12205                 return NULL;
12206         }
12207         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
12208         if (!pool) {
12209                 claim_zero(mlx5_devx_cmd_destroy(obj));
12210                 rte_errno = ENOMEM;
12211                 return NULL;
12212         }
12213         pool->flow_hit_aso_obj = obj;
12214         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
12215         rte_rwlock_write_lock(&mng->resize_rwl);
12216         pool->index = mng->next;
12217         /* Resize pools array if there is no room for the new pool in it. */
12218         if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) {
12219                 claim_zero(mlx5_devx_cmd_destroy(obj));
12220                 mlx5_free(pool);
12221                 rte_rwlock_write_unlock(&mng->resize_rwl);
12222                 return NULL;
12223         }
12224         mng->pools[pool->index] = pool;
12225         mng->next++;
12226         rte_rwlock_write_unlock(&mng->resize_rwl);
12227         /* Assign the first action in the new pool, the rest go to free list. */
12228         *age_free = &pool->actions[0];
12229         for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) {
12230                 pool->actions[i].offset = i;
12231                 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next);
12232         }
12233         return pool;
12234 }
12235
12236 /**
12237  * Allocate a ASO aging bit.
12238  *
12239  * @param[in] dev
12240  *   Pointer to the Ethernet device structure.
12241  * @param[out] error
12242  *   Pointer to the error structure.
12243  *
12244  * @return
12245  *   Index to ASO age action on success, 0 otherwise and rte_errno is set.
12246  */
12247 static uint32_t
12248 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12249 {
12250         struct mlx5_priv *priv = dev->data->dev_private;
12251         const struct mlx5_aso_age_pool *pool;
12252         struct mlx5_aso_age_action *age_free = NULL;
12253         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12254
12255         MLX5_ASSERT(mng);
12256         /* Try to get the next free age action bit. */
12257         rte_spinlock_lock(&mng->free_sl);
12258         age_free = LIST_FIRST(&mng->free);
12259         if (age_free) {
12260                 LIST_REMOVE(age_free, next);
12261         } else if (!flow_dv_age_pool_create(dev, &age_free)) {
12262                 rte_spinlock_unlock(&mng->free_sl);
12263                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12264                                    NULL, "failed to create ASO age pool");
12265                 return 0; /* 0 is an error. */
12266         }
12267         rte_spinlock_unlock(&mng->free_sl);
12268         pool = container_of
12269           ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL])
12270                   (age_free - age_free->offset), const struct mlx5_aso_age_pool,
12271                                                                        actions);
12272         if (!age_free->dr_action) {
12273                 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0,
12274                                                  error);
12275
12276                 if (reg_c < 0) {
12277                         rte_flow_error_set(error, rte_errno,
12278                                            RTE_FLOW_ERROR_TYPE_ACTION,
12279                                            NULL, "failed to get reg_c "
12280                                            "for ASO flow hit");
12281                         return 0; /* 0 is an error. */
12282                 }
12283 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
12284                 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso
12285                                 (priv->sh->rx_domain,
12286                                  pool->flow_hit_aso_obj->obj, age_free->offset,
12287                                  MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET,
12288                                  (reg_c - REG_C_0));
12289 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
12290                 if (!age_free->dr_action) {
12291                         rte_errno = errno;
12292                         rte_spinlock_lock(&mng->free_sl);
12293                         LIST_INSERT_HEAD(&mng->free, age_free, next);
12294                         rte_spinlock_unlock(&mng->free_sl);
12295                         rte_flow_error_set(error, rte_errno,
12296                                            RTE_FLOW_ERROR_TYPE_ACTION,
12297                                            NULL, "failed to create ASO "
12298                                            "flow hit action");
12299                         return 0; /* 0 is an error. */
12300                 }
12301         }
12302         __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED);
12303         return pool->index | ((age_free->offset + 1) << 16);
12304 }
12305
12306 /**
12307  * Initialize flow ASO age parameters.
12308  *
12309  * @param[in] dev
12310  *   Pointer to rte_eth_dev structure.
12311  * @param[in] age_idx
12312  *   Index of ASO age action.
12313  * @param[in] context
12314  *   Pointer to flow counter age context.
12315  * @param[in] timeout
12316  *   Aging timeout in seconds.
12317  *
12318  */
12319 static void
12320 flow_dv_aso_age_params_init(struct rte_eth_dev *dev,
12321                             uint32_t age_idx,
12322                             void *context,
12323                             uint32_t timeout)
12324 {
12325         struct mlx5_aso_age_action *aso_age;
12326
12327         aso_age = flow_aso_age_get_by_idx(dev, age_idx);
12328         MLX5_ASSERT(aso_age);
12329         aso_age->age_params.context = context;
12330         aso_age->age_params.timeout = timeout;
12331         aso_age->age_params.port_id = dev->data->port_id;
12332         __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0,
12333                          __ATOMIC_RELAXED);
12334         __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE,
12335                          __ATOMIC_RELAXED);
12336 }
12337
12338 static void
12339 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask,
12340                                const struct rte_flow_item_integrity *value,
12341                                void *headers_m, void *headers_v)
12342 {
12343         if (mask->l4_ok) {
12344                 /* RTE l4_ok filter aggregates hardware l4_ok and
12345                  * l4_checksum_ok filters.
12346                  * Positive RTE l4_ok match requires hardware match on both L4
12347                  * hardware integrity bits.
12348                  * For negative match, check hardware l4_checksum_ok bit only,
12349                  * because hardware sets that bit to 0 for all packets
12350                  * with bad L4.
12351                  */
12352                 if (value->l4_ok) {
12353                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok, 1);
12354                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok, 1);
12355                 }
12356                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok, 1);
12357                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
12358                          !!value->l4_ok);
12359         }
12360         if (mask->l4_csum_ok) {
12361                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok, 1);
12362                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
12363                          value->l4_csum_ok);
12364         }
12365 }
12366
12367 static void
12368 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask,
12369                                const struct rte_flow_item_integrity *value,
12370                                void *headers_m, void *headers_v, bool is_ipv4)
12371 {
12372         if (mask->l3_ok) {
12373                 /* RTE l3_ok filter aggregates for IPv4 hardware l3_ok and
12374                  * ipv4_csum_ok filters.
12375                  * Positive RTE l3_ok match requires hardware match on both L3
12376                  * hardware integrity bits.
12377                  * For negative match, check hardware l3_csum_ok bit only,
12378                  * because hardware sets that bit to 0 for all packets
12379                  * with bad L3.
12380                  */
12381                 if (is_ipv4) {
12382                         if (value->l3_ok) {
12383                                 MLX5_SET(fte_match_set_lyr_2_4, headers_m,
12384                                          l3_ok, 1);
12385                                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12386                                          l3_ok, 1);
12387                         }
12388                         MLX5_SET(fte_match_set_lyr_2_4, headers_m,
12389                                  ipv4_checksum_ok, 1);
12390                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12391                                  ipv4_checksum_ok, !!value->l3_ok);
12392                 } else {
12393                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok, 1);
12394                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok,
12395                                  value->l3_ok);
12396                 }
12397         }
12398         if (mask->ipv4_csum_ok) {
12399                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok, 1);
12400                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
12401                          value->ipv4_csum_ok);
12402         }
12403 }
12404
12405 static void
12406 set_integrity_bits(void *headers_m, void *headers_v,
12407                    const struct rte_flow_item *integrity_item, bool is_l3_ip4)
12408 {
12409         const struct rte_flow_item_integrity *spec = integrity_item->spec;
12410         const struct rte_flow_item_integrity *mask = integrity_item->mask;
12411
12412         /* Integrity bits validation cleared spec pointer */
12413         MLX5_ASSERT(spec != NULL);
12414         if (!mask)
12415                 mask = &rte_flow_item_integrity_mask;
12416         flow_dv_translate_integrity_l3(mask, spec, headers_m, headers_v,
12417                                        is_l3_ip4);
12418         flow_dv_translate_integrity_l4(mask, spec, headers_m, headers_v);
12419 }
12420
12421 static void
12422 flow_dv_translate_item_integrity_post(void *matcher, void *key,
12423                                       const
12424                                       struct rte_flow_item *integrity_items[2],
12425                                       uint64_t pattern_flags)
12426 {
12427         void *headers_m, *headers_v;
12428         bool is_l3_ip4;
12429
12430         if (pattern_flags & MLX5_FLOW_ITEM_INNER_INTEGRITY) {
12431                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12432                                          inner_headers);
12433                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
12434                 is_l3_ip4 = (pattern_flags & MLX5_FLOW_LAYER_INNER_L3_IPV4) !=
12435                             0;
12436                 set_integrity_bits(headers_m, headers_v,
12437                                    integrity_items[1], is_l3_ip4);
12438         }
12439         if (pattern_flags & MLX5_FLOW_ITEM_OUTER_INTEGRITY) {
12440                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12441                                          outer_headers);
12442                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
12443                 is_l3_ip4 = (pattern_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) !=
12444                             0;
12445                 set_integrity_bits(headers_m, headers_v,
12446                                    integrity_items[0], is_l3_ip4);
12447         }
12448 }
12449
12450 static void
12451 flow_dv_translate_item_integrity(const struct rte_flow_item *item,
12452                                  const struct rte_flow_item *integrity_items[2],
12453                                  uint64_t *last_item)
12454 {
12455         const struct rte_flow_item_integrity *spec = (typeof(spec))item->spec;
12456
12457         /* integrity bits validation cleared spec pointer */
12458         MLX5_ASSERT(spec != NULL);
12459         if (spec->level > 1) {
12460                 integrity_items[1] = item;
12461                 *last_item |= MLX5_FLOW_ITEM_INNER_INTEGRITY;
12462         } else {
12463                 integrity_items[0] = item;
12464                 *last_item |= MLX5_FLOW_ITEM_OUTER_INTEGRITY;
12465         }
12466 }
12467
12468 /**
12469  * Prepares DV flow counter with aging configuration.
12470  * Gets it by index when exists, creates a new one when doesn't.
12471  *
12472  * @param[in] dev
12473  *   Pointer to rte_eth_dev structure.
12474  * @param[in] dev_flow
12475  *   Pointer to the mlx5_flow.
12476  * @param[in, out] flow
12477  *   Pointer to the sub flow.
12478  * @param[in] count
12479  *   Pointer to the counter action configuration.
12480  * @param[in] age
12481  *   Pointer to the aging action configuration.
12482  * @param[out] error
12483  *   Pointer to the error structure.
12484  *
12485  * @return
12486  *   Pointer to the counter, NULL otherwise.
12487  */
12488 static struct mlx5_flow_counter *
12489 flow_dv_prepare_counter(struct rte_eth_dev *dev,
12490                         struct mlx5_flow *dev_flow,
12491                         struct rte_flow *flow,
12492                         const struct rte_flow_action_count *count,
12493                         const struct rte_flow_action_age *age,
12494                         struct rte_flow_error *error)
12495 {
12496         if (!flow->counter) {
12497                 flow->counter = flow_dv_translate_create_counter(dev, dev_flow,
12498                                                                  count, age);
12499                 if (!flow->counter) {
12500                         rte_flow_error_set(error, rte_errno,
12501                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12502                                            "cannot create counter object.");
12503                         return NULL;
12504                 }
12505         }
12506         return flow_dv_counter_get_by_idx(dev, flow->counter, NULL);
12507 }
12508
12509 /*
12510  * Release an ASO CT action by its own device.
12511  *
12512  * @param[in] dev
12513  *   Pointer to the Ethernet device structure.
12514  * @param[in] idx
12515  *   Index of ASO CT action to release.
12516  *
12517  * @return
12518  *   0 when CT action was removed, otherwise the number of references.
12519  */
12520 static inline int
12521 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx)
12522 {
12523         struct mlx5_priv *priv = dev->data->dev_private;
12524         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12525         uint32_t ret;
12526         struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12527         enum mlx5_aso_ct_state state =
12528                         __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
12529
12530         /* Cannot release when CT is in the ASO SQ. */
12531         if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY)
12532                 return -1;
12533         ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED);
12534         if (!ret) {
12535                 if (ct->dr_action_orig) {
12536 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12537                         claim_zero(mlx5_glue->destroy_flow_action
12538                                         (ct->dr_action_orig));
12539 #endif
12540                         ct->dr_action_orig = NULL;
12541                 }
12542                 if (ct->dr_action_rply) {
12543 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12544                         claim_zero(mlx5_glue->destroy_flow_action
12545                                         (ct->dr_action_rply));
12546 #endif
12547                         ct->dr_action_rply = NULL;
12548                 }
12549                 /* Clear the state to free, no need in 1st allocation. */
12550                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE);
12551                 rte_spinlock_lock(&mng->ct_sl);
12552                 LIST_INSERT_HEAD(&mng->free_cts, ct, next);
12553                 rte_spinlock_unlock(&mng->ct_sl);
12554         }
12555         return (int)ret;
12556 }
12557
12558 static inline int
12559 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx,
12560                        struct rte_flow_error *error)
12561 {
12562         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx);
12563         uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx);
12564         struct rte_eth_dev *owndev = &rte_eth_devices[owner];
12565         int ret;
12566
12567         MLX5_ASSERT(owner < RTE_MAX_ETHPORTS);
12568         if (dev->data->dev_started != 1)
12569                 return rte_flow_error_set(error, EAGAIN,
12570                                           RTE_FLOW_ERROR_TYPE_ACTION,
12571                                           NULL,
12572                                           "Indirect CT action cannot be destroyed when the port is stopped");
12573         ret = flow_dv_aso_ct_dev_release(owndev, idx);
12574         if (ret < 0)
12575                 return rte_flow_error_set(error, EAGAIN,
12576                                           RTE_FLOW_ERROR_TYPE_ACTION,
12577                                           NULL,
12578                                           "Current state prevents indirect CT action from being destroyed");
12579         return ret;
12580 }
12581
12582 /*
12583  * Resize the ASO CT pools array by 64 pools.
12584  *
12585  * @param[in] dev
12586  *   Pointer to the Ethernet device structure.
12587  *
12588  * @return
12589  *   0 on success, otherwise negative errno value and rte_errno is set.
12590  */
12591 static int
12592 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev)
12593 {
12594         struct mlx5_priv *priv = dev->data->dev_private;
12595         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12596         void *old_pools = mng->pools;
12597         /* Magic number now, need a macro. */
12598         uint32_t resize = mng->n + 64;
12599         uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize;
12600         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
12601
12602         if (!pools) {
12603                 rte_errno = ENOMEM;
12604                 return -rte_errno;
12605         }
12606         rte_rwlock_write_lock(&mng->resize_rwl);
12607         /* ASO SQ/QP was already initialized in the startup. */
12608         if (old_pools) {
12609                 /* Realloc could be an alternative choice. */
12610                 rte_memcpy(pools, old_pools,
12611                            mng->n * sizeof(struct mlx5_aso_ct_pool *));
12612                 mlx5_free(old_pools);
12613         }
12614         mng->n = resize;
12615         mng->pools = pools;
12616         rte_rwlock_write_unlock(&mng->resize_rwl);
12617         return 0;
12618 }
12619
12620 /*
12621  * Create and initialize a new ASO CT pool.
12622  *
12623  * @param[in] dev
12624  *   Pointer to the Ethernet device structure.
12625  * @param[out] ct_free
12626  *   Where to put the pointer of a new CT action.
12627  *
12628  * @return
12629  *   The CT actions pool pointer and @p ct_free is set on success,
12630  *   NULL otherwise and rte_errno is set.
12631  */
12632 static struct mlx5_aso_ct_pool *
12633 flow_dv_ct_pool_create(struct rte_eth_dev *dev,
12634                        struct mlx5_aso_ct_action **ct_free)
12635 {
12636         struct mlx5_priv *priv = dev->data->dev_private;
12637         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12638         struct mlx5_aso_ct_pool *pool = NULL;
12639         struct mlx5_devx_obj *obj = NULL;
12640         uint32_t i;
12641         uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL);
12642
12643         obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->cdev->ctx,
12644                                                           priv->sh->cdev->pdn,
12645                                                           log_obj_size);
12646         if (!obj) {
12647                 rte_errno = ENODATA;
12648                 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
12649                 return NULL;
12650         }
12651         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
12652         if (!pool) {
12653                 rte_errno = ENOMEM;
12654                 claim_zero(mlx5_devx_cmd_destroy(obj));
12655                 return NULL;
12656         }
12657         pool->devx_obj = obj;
12658         pool->index = mng->next;
12659         /* Resize pools array if there is no room for the new pool in it. */
12660         if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) {
12661                 claim_zero(mlx5_devx_cmd_destroy(obj));
12662                 mlx5_free(pool);
12663                 return NULL;
12664         }
12665         mng->pools[pool->index] = pool;
12666         mng->next++;
12667         /* Assign the first action in the new pool, the rest go to free list. */
12668         *ct_free = &pool->actions[0];
12669         /* Lock outside, the list operation is safe here. */
12670         for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) {
12671                 /* refcnt is 0 when allocating the memory. */
12672                 pool->actions[i].offset = i;
12673                 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next);
12674         }
12675         return pool;
12676 }
12677
12678 /*
12679  * Allocate a ASO CT action from free list.
12680  *
12681  * @param[in] dev
12682  *   Pointer to the Ethernet device structure.
12683  * @param[out] error
12684  *   Pointer to the error structure.
12685  *
12686  * @return
12687  *   Index to ASO CT action on success, 0 otherwise and rte_errno is set.
12688  */
12689 static uint32_t
12690 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12691 {
12692         struct mlx5_priv *priv = dev->data->dev_private;
12693         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12694         struct mlx5_aso_ct_action *ct = NULL;
12695         struct mlx5_aso_ct_pool *pool;
12696         uint8_t reg_c;
12697         uint32_t ct_idx;
12698
12699         MLX5_ASSERT(mng);
12700         if (!priv->sh->cdev->config.devx) {
12701                 rte_errno = ENOTSUP;
12702                 return 0;
12703         }
12704         /* Get a free CT action, if no, a new pool will be created. */
12705         rte_spinlock_lock(&mng->ct_sl);
12706         ct = LIST_FIRST(&mng->free_cts);
12707         if (ct) {
12708                 LIST_REMOVE(ct, next);
12709         } else if (!flow_dv_ct_pool_create(dev, &ct)) {
12710                 rte_spinlock_unlock(&mng->ct_sl);
12711                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12712                                    NULL, "failed to create ASO CT pool");
12713                 return 0;
12714         }
12715         rte_spinlock_unlock(&mng->ct_sl);
12716         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
12717         ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset);
12718         /* 0: inactive, 1: created, 2+: used by flows. */
12719         __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED);
12720         reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error);
12721         if (!ct->dr_action_orig) {
12722 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12723                 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso
12724                         (priv->sh->rx_domain, pool->devx_obj->obj,
12725                          ct->offset,
12726                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR,
12727                          reg_c - REG_C_0);
12728 #else
12729                 RTE_SET_USED(reg_c);
12730 #endif
12731                 if (!ct->dr_action_orig) {
12732                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12733                         rte_flow_error_set(error, rte_errno,
12734                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12735                                            "failed to create ASO CT action");
12736                         return 0;
12737                 }
12738         }
12739         if (!ct->dr_action_rply) {
12740 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12741                 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso
12742                         (priv->sh->rx_domain, pool->devx_obj->obj,
12743                          ct->offset,
12744                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER,
12745                          reg_c - REG_C_0);
12746 #endif
12747                 if (!ct->dr_action_rply) {
12748                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12749                         rte_flow_error_set(error, rte_errno,
12750                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12751                                            "failed to create ASO CT action");
12752                         return 0;
12753                 }
12754         }
12755         return ct_idx;
12756 }
12757
12758 /*
12759  * Create a conntrack object with context and actions by using ASO mechanism.
12760  *
12761  * @param[in] dev
12762  *   Pointer to rte_eth_dev structure.
12763  * @param[in] pro
12764  *   Pointer to conntrack information profile.
12765  * @param[out] error
12766  *   Pointer to the error structure.
12767  *
12768  * @return
12769  *   Index to conntrack object on success, 0 otherwise.
12770  */
12771 static uint32_t
12772 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev,
12773                                    const struct rte_flow_action_conntrack *pro,
12774                                    struct rte_flow_error *error)
12775 {
12776         struct mlx5_priv *priv = dev->data->dev_private;
12777         struct mlx5_dev_ctx_shared *sh = priv->sh;
12778         struct mlx5_aso_ct_action *ct;
12779         uint32_t idx;
12780
12781         if (!sh->ct_aso_en)
12782                 return rte_flow_error_set(error, ENOTSUP,
12783                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12784                                           "Connection is not supported");
12785         idx = flow_dv_aso_ct_alloc(dev, error);
12786         if (!idx)
12787                 return rte_flow_error_set(error, rte_errno,
12788                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12789                                           "Failed to allocate CT object");
12790         ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12791         if (mlx5_aso_ct_update_by_wqe(sh, ct, pro))
12792                 return rte_flow_error_set(error, EBUSY,
12793                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12794                                           "Failed to update CT");
12795         ct->is_original = !!pro->is_original_dir;
12796         ct->peer = pro->peer_port;
12797         return idx;
12798 }
12799
12800 /**
12801  * Fill the flow with DV spec, lock free
12802  * (mutex should be acquired by caller).
12803  *
12804  * @param[in] dev
12805  *   Pointer to rte_eth_dev structure.
12806  * @param[in, out] dev_flow
12807  *   Pointer to the sub flow.
12808  * @param[in] attr
12809  *   Pointer to the flow attributes.
12810  * @param[in] items
12811  *   Pointer to the list of items.
12812  * @param[in] actions
12813  *   Pointer to the list of actions.
12814  * @param[out] error
12815  *   Pointer to the error structure.
12816  *
12817  * @return
12818  *   0 on success, a negative errno value otherwise and rte_errno is set.
12819  */
12820 static int
12821 flow_dv_translate(struct rte_eth_dev *dev,
12822                   struct mlx5_flow *dev_flow,
12823                   const struct rte_flow_attr *attr,
12824                   const struct rte_flow_item items[],
12825                   const struct rte_flow_action actions[],
12826                   struct rte_flow_error *error)
12827 {
12828         struct mlx5_priv *priv = dev->data->dev_private;
12829         struct mlx5_sh_config *dev_conf = &priv->sh->config;
12830         struct rte_flow *flow = dev_flow->flow;
12831         struct mlx5_flow_handle *handle = dev_flow->handle;
12832         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12833         struct mlx5_flow_rss_desc *rss_desc;
12834         uint64_t item_flags = 0;
12835         uint64_t last_item = 0;
12836         uint64_t action_flags = 0;
12837         struct mlx5_flow_dv_matcher matcher = {
12838                 .mask = {
12839                         .size = sizeof(matcher.mask.buf),
12840                 },
12841         };
12842         int actions_n = 0;
12843         bool actions_end = false;
12844         union {
12845                 struct mlx5_flow_dv_modify_hdr_resource res;
12846                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
12847                             sizeof(struct mlx5_modification_cmd) *
12848                             (MLX5_MAX_MODIFY_NUM + 1)];
12849         } mhdr_dummy;
12850         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
12851         const struct rte_flow_action_count *count = NULL;
12852         const struct rte_flow_action_age *non_shared_age = NULL;
12853         union flow_dv_attr flow_attr = { .attr = 0 };
12854         uint32_t tag_be;
12855         union mlx5_flow_tbl_key tbl_key;
12856         uint32_t modify_action_position = UINT32_MAX;
12857         void *match_mask = matcher.mask.buf;
12858         void *match_value = dev_flow->dv.value.buf;
12859         uint8_t next_protocol = 0xff;
12860         struct rte_vlan_hdr vlan = { 0 };
12861         struct mlx5_flow_dv_dest_array_resource mdest_res;
12862         struct mlx5_flow_dv_sample_resource sample_res;
12863         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
12864         const struct rte_flow_action_sample *sample = NULL;
12865         struct mlx5_flow_sub_actions_list *sample_act;
12866         uint32_t sample_act_pos = UINT32_MAX;
12867         uint32_t age_act_pos = UINT32_MAX;
12868         uint32_t num_of_dest = 0;
12869         int tmp_actions_n = 0;
12870         uint32_t table;
12871         int ret = 0;
12872         const struct mlx5_flow_tunnel *tunnel = NULL;
12873         struct flow_grp_info grp_info = {
12874                 .external = !!dev_flow->external,
12875                 .transfer = !!attr->transfer,
12876                 .fdb_def_rule = !!priv->fdb_def_rule,
12877                 .skip_scale = dev_flow->skip_scale &
12878                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
12879                 .std_tbl_fix = true,
12880         };
12881         const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
12882         const struct rte_flow_item *tunnel_item = NULL;
12883         const struct rte_flow_item *gre_item = NULL;
12884
12885         if (!wks)
12886                 return rte_flow_error_set(error, ENOMEM,
12887                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12888                                           NULL,
12889                                           "failed to push flow workspace");
12890         rss_desc = &wks->rss_desc;
12891         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
12892         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
12893         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12894                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12895         /* update normal path action resource into last index of array */
12896         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
12897         if (is_tunnel_offload_active(dev)) {
12898                 if (dev_flow->tunnel) {
12899                         RTE_VERIFY(dev_flow->tof_type ==
12900                                    MLX5_TUNNEL_OFFLOAD_MISS_RULE);
12901                         tunnel = dev_flow->tunnel;
12902                 } else {
12903                         tunnel = mlx5_get_tof(items, actions,
12904                                               &dev_flow->tof_type);
12905                         dev_flow->tunnel = tunnel;
12906                 }
12907                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
12908                                         (dev, attr, tunnel, dev_flow->tof_type);
12909         }
12910         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12911                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12912         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
12913                                        &grp_info, error);
12914         if (ret)
12915                 return ret;
12916         dev_flow->dv.group = table;
12917         if (attr->transfer)
12918                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12919         /* number of actions must be set to 0 in case of dirty stack. */
12920         mhdr_res->actions_num = 0;
12921         if (is_flow_tunnel_match_rule(dev_flow->tof_type)) {
12922                 /*
12923                  * do not add decap action if match rule drops packet
12924                  * HW rejects rules with decap & drop
12925                  *
12926                  * if tunnel match rule was inserted before matching tunnel set
12927                  * rule flow table used in the match rule must be registered.
12928                  * current implementation handles that in the
12929                  * flow_dv_match_register() at the function end.
12930                  */
12931                 bool add_decap = true;
12932                 const struct rte_flow_action *ptr = actions;
12933
12934                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
12935                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
12936                                 add_decap = false;
12937                                 break;
12938                         }
12939                 }
12940                 if (add_decap) {
12941                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12942                                                            attr->transfer,
12943                                                            error))
12944                                 return -rte_errno;
12945                         dev_flow->dv.actions[actions_n++] =
12946                                         dev_flow->dv.encap_decap->action;
12947                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12948                 }
12949         }
12950         for (; !actions_end ; actions++) {
12951                 const struct rte_flow_action_queue *queue;
12952                 const struct rte_flow_action_rss *rss;
12953                 const struct rte_flow_action *action = actions;
12954                 const uint8_t *rss_key;
12955                 struct mlx5_flow_tbl_resource *tbl;
12956                 struct mlx5_aso_age_action *age_act;
12957                 struct mlx5_flow_counter *cnt_act;
12958                 uint32_t port_id = 0;
12959                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
12960                 int action_type = actions->type;
12961                 const struct rte_flow_action *found_action = NULL;
12962                 uint32_t jump_group = 0;
12963                 uint32_t owner_idx;
12964                 struct mlx5_aso_ct_action *ct;
12965
12966                 if (!mlx5_flow_os_action_supported(action_type))
12967                         return rte_flow_error_set(error, ENOTSUP,
12968                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12969                                                   actions,
12970                                                   "action not supported");
12971                 switch (action_type) {
12972                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
12973                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
12974                         break;
12975                 case RTE_FLOW_ACTION_TYPE_VOID:
12976                         break;
12977                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
12978                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
12979                         if (flow_dv_translate_action_port_id(dev, action,
12980                                                              &port_id, error))
12981                                 return -rte_errno;
12982                         port_id_resource.port_id = port_id;
12983                         MLX5_ASSERT(!handle->rix_port_id_action);
12984                         if (flow_dv_port_id_action_resource_register
12985                             (dev, &port_id_resource, dev_flow, error))
12986                                 return -rte_errno;
12987                         dev_flow->dv.actions[actions_n++] =
12988                                         dev_flow->dv.port_id_action->action;
12989                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12990                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
12991                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12992                         num_of_dest++;
12993                         break;
12994                 case RTE_FLOW_ACTION_TYPE_FLAG:
12995                         action_flags |= MLX5_FLOW_ACTION_FLAG;
12996                         wks->mark = 1;
12997                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12998                                 struct rte_flow_action_mark mark = {
12999                                         .id = MLX5_FLOW_MARK_DEFAULT,
13000                                 };
13001
13002                                 if (flow_dv_convert_action_mark(dev, &mark,
13003                                                                 mhdr_res,
13004                                                                 error))
13005                                         return -rte_errno;
13006                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
13007                                 break;
13008                         }
13009                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
13010                         /*
13011                          * Only one FLAG or MARK is supported per device flow
13012                          * right now. So the pointer to the tag resource must be
13013                          * zero before the register process.
13014                          */
13015                         MLX5_ASSERT(!handle->dvh.rix_tag);
13016                         if (flow_dv_tag_resource_register(dev, tag_be,
13017                                                           dev_flow, error))
13018                                 return -rte_errno;
13019                         MLX5_ASSERT(dev_flow->dv.tag_resource);
13020                         dev_flow->dv.actions[actions_n++] =
13021                                         dev_flow->dv.tag_resource->action;
13022                         break;
13023                 case RTE_FLOW_ACTION_TYPE_MARK:
13024                         action_flags |= MLX5_FLOW_ACTION_MARK;
13025                         wks->mark = 1;
13026                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
13027                                 const struct rte_flow_action_mark *mark =
13028                                         (const struct rte_flow_action_mark *)
13029                                                 actions->conf;
13030
13031                                 if (flow_dv_convert_action_mark(dev, mark,
13032                                                                 mhdr_res,
13033                                                                 error))
13034                                         return -rte_errno;
13035                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
13036                                 break;
13037                         }
13038                         /* Fall-through */
13039                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
13040                         /* Legacy (non-extensive) MARK action. */
13041                         tag_be = mlx5_flow_mark_set
13042                               (((const struct rte_flow_action_mark *)
13043                                (actions->conf))->id);
13044                         MLX5_ASSERT(!handle->dvh.rix_tag);
13045                         if (flow_dv_tag_resource_register(dev, tag_be,
13046                                                           dev_flow, error))
13047                                 return -rte_errno;
13048                         MLX5_ASSERT(dev_flow->dv.tag_resource);
13049                         dev_flow->dv.actions[actions_n++] =
13050                                         dev_flow->dv.tag_resource->action;
13051                         break;
13052                 case RTE_FLOW_ACTION_TYPE_SET_META:
13053                         if (flow_dv_convert_action_set_meta
13054                                 (dev, mhdr_res, attr,
13055                                  (const struct rte_flow_action_set_meta *)
13056                                   actions->conf, error))
13057                                 return -rte_errno;
13058                         action_flags |= MLX5_FLOW_ACTION_SET_META;
13059                         break;
13060                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
13061                         if (flow_dv_convert_action_set_tag
13062                                 (dev, mhdr_res,
13063                                  (const struct rte_flow_action_set_tag *)
13064                                   actions->conf, error))
13065                                 return -rte_errno;
13066                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13067                         break;
13068                 case RTE_FLOW_ACTION_TYPE_DROP:
13069                         action_flags |= MLX5_FLOW_ACTION_DROP;
13070                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
13071                         break;
13072                 case RTE_FLOW_ACTION_TYPE_QUEUE:
13073                         queue = actions->conf;
13074                         rss_desc->queue_num = 1;
13075                         rss_desc->queue[0] = queue->index;
13076                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
13077                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
13078                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
13079                         num_of_dest++;
13080                         break;
13081                 case RTE_FLOW_ACTION_TYPE_RSS:
13082                         rss = actions->conf;
13083                         memcpy(rss_desc->queue, rss->queue,
13084                                rss->queue_num * sizeof(uint16_t));
13085                         rss_desc->queue_num = rss->queue_num;
13086                         /* NULL RSS key indicates default RSS key. */
13087                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
13088                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
13089                         /*
13090                          * rss->level and rss.types should be set in advance
13091                          * when expanding items for RSS.
13092                          */
13093                         action_flags |= MLX5_FLOW_ACTION_RSS;
13094                         dev_flow->handle->fate_action = rss_desc->shared_rss ?
13095                                 MLX5_FLOW_FATE_SHARED_RSS :
13096                                 MLX5_FLOW_FATE_QUEUE;
13097                         break;
13098                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
13099                         owner_idx = (uint32_t)(uintptr_t)action->conf;
13100                         age_act = flow_aso_age_get_by_idx(dev, owner_idx);
13101                         if (flow->age == 0) {
13102                                 flow->age = owner_idx;
13103                                 __atomic_fetch_add(&age_act->refcnt, 1,
13104                                                    __ATOMIC_RELAXED);
13105                         }
13106                         age_act_pos = actions_n++;
13107                         action_flags |= MLX5_FLOW_ACTION_AGE;
13108                         break;
13109                 case RTE_FLOW_ACTION_TYPE_AGE:
13110                         non_shared_age = action->conf;
13111                         age_act_pos = actions_n++;
13112                         action_flags |= MLX5_FLOW_ACTION_AGE;
13113                         break;
13114                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
13115                         owner_idx = (uint32_t)(uintptr_t)action->conf;
13116                         cnt_act = flow_dv_counter_get_by_idx(dev, owner_idx,
13117                                                              NULL);
13118                         MLX5_ASSERT(cnt_act != NULL);
13119                         /**
13120                          * When creating meter drop flow in drop table, the
13121                          * counter should not overwrite the rte flow counter.
13122                          */
13123                         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
13124                             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP) {
13125                                 dev_flow->dv.actions[actions_n++] =
13126                                                         cnt_act->action;
13127                         } else {
13128                                 if (flow->counter == 0) {
13129                                         flow->counter = owner_idx;
13130                                         __atomic_fetch_add
13131                                                 (&cnt_act->shared_info.refcnt,
13132                                                  1, __ATOMIC_RELAXED);
13133                                 }
13134                                 /* Save information first, will apply later. */
13135                                 action_flags |= MLX5_FLOW_ACTION_COUNT;
13136                         }
13137                         break;
13138                 case RTE_FLOW_ACTION_TYPE_COUNT:
13139                         if (!priv->sh->cdev->config.devx) {
13140                                 return rte_flow_error_set
13141                                               (error, ENOTSUP,
13142                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
13143                                                NULL,
13144                                                "count action not supported");
13145                         }
13146                         /* Save information first, will apply later. */
13147                         count = action->conf;
13148                         action_flags |= MLX5_FLOW_ACTION_COUNT;
13149                         break;
13150                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
13151                         dev_flow->dv.actions[actions_n++] =
13152                                                 priv->sh->pop_vlan_action;
13153                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
13154                         break;
13155                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
13156                         if (!(action_flags &
13157                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
13158                                 flow_dev_get_vlan_info_from_items(items, &vlan);
13159                         vlan.eth_proto = rte_be_to_cpu_16
13160                              ((((const struct rte_flow_action_of_push_vlan *)
13161                                                    actions->conf)->ethertype));
13162                         found_action = mlx5_flow_find_action
13163                                         (actions + 1,
13164                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
13165                         if (found_action)
13166                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
13167                         found_action = mlx5_flow_find_action
13168                                         (actions + 1,
13169                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
13170                         if (found_action)
13171                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
13172                         if (flow_dv_create_action_push_vlan
13173                                             (dev, attr, &vlan, dev_flow, error))
13174                                 return -rte_errno;
13175                         dev_flow->dv.actions[actions_n++] =
13176                                         dev_flow->dv.push_vlan_res->action;
13177                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
13178                         break;
13179                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
13180                         /* of_vlan_push action handled this action */
13181                         MLX5_ASSERT(action_flags &
13182                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
13183                         break;
13184                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
13185                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
13186                                 break;
13187                         flow_dev_get_vlan_info_from_items(items, &vlan);
13188                         mlx5_update_vlan_vid_pcp(actions, &vlan);
13189                         /* If no VLAN push - this is a modify header action */
13190                         if (flow_dv_convert_action_modify_vlan_vid
13191                                                 (mhdr_res, actions, error))
13192                                 return -rte_errno;
13193                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
13194                         break;
13195                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
13196                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
13197                         if (flow_dv_create_action_l2_encap(dev, actions,
13198                                                            dev_flow,
13199                                                            attr->transfer,
13200                                                            error))
13201                                 return -rte_errno;
13202                         dev_flow->dv.actions[actions_n++] =
13203                                         dev_flow->dv.encap_decap->action;
13204                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
13205                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
13206                                 sample_act->action_flags |=
13207                                                         MLX5_FLOW_ACTION_ENCAP;
13208                         break;
13209                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
13210                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
13211                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
13212                                                            attr->transfer,
13213                                                            error))
13214                                 return -rte_errno;
13215                         dev_flow->dv.actions[actions_n++] =
13216                                         dev_flow->dv.encap_decap->action;
13217                         action_flags |= MLX5_FLOW_ACTION_DECAP;
13218                         break;
13219                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
13220                         /* Handle encap with preceding decap. */
13221                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
13222                                 if (flow_dv_create_action_raw_encap
13223                                         (dev, actions, dev_flow, attr, error))
13224                                         return -rte_errno;
13225                                 dev_flow->dv.actions[actions_n++] =
13226                                         dev_flow->dv.encap_decap->action;
13227                         } else {
13228                                 /* Handle encap without preceding decap. */
13229                                 if (flow_dv_create_action_l2_encap
13230                                     (dev, actions, dev_flow, attr->transfer,
13231                                      error))
13232                                         return -rte_errno;
13233                                 dev_flow->dv.actions[actions_n++] =
13234                                         dev_flow->dv.encap_decap->action;
13235                         }
13236                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
13237                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
13238                                 sample_act->action_flags |=
13239                                                         MLX5_FLOW_ACTION_ENCAP;
13240                         break;
13241                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
13242                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
13243                                 ;
13244                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
13245                                 if (flow_dv_create_action_l2_decap
13246                                     (dev, dev_flow, attr->transfer, error))
13247                                         return -rte_errno;
13248                                 dev_flow->dv.actions[actions_n++] =
13249                                         dev_flow->dv.encap_decap->action;
13250                         }
13251                         /* If decap is followed by encap, handle it at encap. */
13252                         action_flags |= MLX5_FLOW_ACTION_DECAP;
13253                         break;
13254                 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP:
13255                         dev_flow->dv.actions[actions_n++] =
13256                                 (void *)(uintptr_t)action->conf;
13257                         action_flags |= MLX5_FLOW_ACTION_JUMP;
13258                         break;
13259                 case RTE_FLOW_ACTION_TYPE_JUMP:
13260                         jump_group = ((const struct rte_flow_action_jump *)
13261                                                         action->conf)->group;
13262                         grp_info.std_tbl_fix = 0;
13263                         if (dev_flow->skip_scale &
13264                                 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT))
13265                                 grp_info.skip_scale = 1;
13266                         else
13267                                 grp_info.skip_scale = 0;
13268                         ret = mlx5_flow_group_to_table(dev, tunnel,
13269                                                        jump_group,
13270                                                        &table,
13271                                                        &grp_info, error);
13272                         if (ret)
13273                                 return ret;
13274                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
13275                                                        attr->transfer,
13276                                                        !!dev_flow->external,
13277                                                        tunnel, jump_group, 0,
13278                                                        0, error);
13279                         if (!tbl)
13280                                 return rte_flow_error_set
13281                                                 (error, errno,
13282                                                  RTE_FLOW_ERROR_TYPE_ACTION,
13283                                                  NULL,
13284                                                  "cannot create jump action.");
13285                         if (flow_dv_jump_tbl_resource_register
13286                             (dev, tbl, dev_flow, error)) {
13287                                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
13288                                 return rte_flow_error_set
13289                                                 (error, errno,
13290                                                  RTE_FLOW_ERROR_TYPE_ACTION,
13291                                                  NULL,
13292                                                  "cannot create jump action.");
13293                         }
13294                         dev_flow->dv.actions[actions_n++] =
13295                                         dev_flow->dv.jump->action;
13296                         action_flags |= MLX5_FLOW_ACTION_JUMP;
13297                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
13298                         sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP;
13299                         num_of_dest++;
13300                         break;
13301                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
13302                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
13303                         if (flow_dv_convert_action_modify_mac
13304                                         (mhdr_res, actions, error))
13305                                 return -rte_errno;
13306                         action_flags |= actions->type ==
13307                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
13308                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
13309                                         MLX5_FLOW_ACTION_SET_MAC_DST;
13310                         break;
13311                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
13312                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
13313                         if (flow_dv_convert_action_modify_ipv4
13314                                         (mhdr_res, actions, error))
13315                                 return -rte_errno;
13316                         action_flags |= actions->type ==
13317                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
13318                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
13319                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
13320                         break;
13321                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
13322                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
13323                         if (flow_dv_convert_action_modify_ipv6
13324                                         (mhdr_res, actions, error))
13325                                 return -rte_errno;
13326                         action_flags |= actions->type ==
13327                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
13328                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
13329                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
13330                         break;
13331                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
13332                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
13333                         if (flow_dv_convert_action_modify_tp
13334                                         (mhdr_res, actions, items,
13335                                          &flow_attr, dev_flow, !!(action_flags &
13336                                          MLX5_FLOW_ACTION_DECAP), error))
13337                                 return -rte_errno;
13338                         action_flags |= actions->type ==
13339                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
13340                                         MLX5_FLOW_ACTION_SET_TP_SRC :
13341                                         MLX5_FLOW_ACTION_SET_TP_DST;
13342                         break;
13343                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
13344                         if (flow_dv_convert_action_modify_dec_ttl
13345                                         (mhdr_res, items, &flow_attr, dev_flow,
13346                                          !!(action_flags &
13347                                          MLX5_FLOW_ACTION_DECAP), error))
13348                                 return -rte_errno;
13349                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
13350                         break;
13351                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
13352                         if (flow_dv_convert_action_modify_ttl
13353                                         (mhdr_res, actions, items, &flow_attr,
13354                                          dev_flow, !!(action_flags &
13355                                          MLX5_FLOW_ACTION_DECAP), error))
13356                                 return -rte_errno;
13357                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
13358                         break;
13359                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
13360                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
13361                         if (flow_dv_convert_action_modify_tcp_seq
13362                                         (mhdr_res, actions, error))
13363                                 return -rte_errno;
13364                         action_flags |= actions->type ==
13365                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
13366                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
13367                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
13368                         break;
13369
13370                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
13371                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
13372                         if (flow_dv_convert_action_modify_tcp_ack
13373                                         (mhdr_res, actions, error))
13374                                 return -rte_errno;
13375                         action_flags |= actions->type ==
13376                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
13377                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
13378                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
13379                         break;
13380                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
13381                         if (flow_dv_convert_action_set_reg
13382                                         (mhdr_res, actions, error))
13383                                 return -rte_errno;
13384                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13385                         break;
13386                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
13387                         if (flow_dv_convert_action_copy_mreg
13388                                         (dev, mhdr_res, actions, error))
13389                                 return -rte_errno;
13390                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13391                         break;
13392                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
13393                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
13394                         dev_flow->handle->fate_action =
13395                                         MLX5_FLOW_FATE_DEFAULT_MISS;
13396                         break;
13397                 case RTE_FLOW_ACTION_TYPE_METER:
13398                         if (!wks->fm)
13399                                 return rte_flow_error_set(error, rte_errno,
13400                                         RTE_FLOW_ERROR_TYPE_ACTION,
13401                                         NULL, "Failed to get meter in flow.");
13402                         /* Set the meter action. */
13403                         dev_flow->dv.actions[actions_n++] =
13404                                 wks->fm->meter_action;
13405                         action_flags |= MLX5_FLOW_ACTION_METER;
13406                         break;
13407                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
13408                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
13409                                                               actions, error))
13410                                 return -rte_errno;
13411                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
13412                         break;
13413                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
13414                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
13415                                                               actions, error))
13416                                 return -rte_errno;
13417                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
13418                         break;
13419                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
13420                         sample_act_pos = actions_n;
13421                         sample = (const struct rte_flow_action_sample *)
13422                                  action->conf;
13423                         actions_n++;
13424                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
13425                         /* put encap action into group if work with port id */
13426                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
13427                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
13428                                 sample_act->action_flags |=
13429                                                         MLX5_FLOW_ACTION_ENCAP;
13430                         break;
13431                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
13432                         if (flow_dv_convert_action_modify_field
13433                                         (dev, mhdr_res, actions, attr, error))
13434                                 return -rte_errno;
13435                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
13436                         break;
13437                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
13438                         owner_idx = (uint32_t)(uintptr_t)action->conf;
13439                         ct = flow_aso_ct_get_by_idx(dev, owner_idx);
13440                         if (!ct)
13441                                 return rte_flow_error_set(error, EINVAL,
13442                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13443                                                 NULL,
13444                                                 "Failed to get CT object.");
13445                         if (mlx5_aso_ct_available(priv->sh, ct))
13446                                 return rte_flow_error_set(error, rte_errno,
13447                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13448                                                 NULL,
13449                                                 "CT is unavailable.");
13450                         if (ct->is_original)
13451                                 dev_flow->dv.actions[actions_n] =
13452                                                         ct->dr_action_orig;
13453                         else
13454                                 dev_flow->dv.actions[actions_n] =
13455                                                         ct->dr_action_rply;
13456                         if (flow->ct == 0) {
13457                                 flow->indirect_type =
13458                                                 MLX5_INDIRECT_ACTION_TYPE_CT;
13459                                 flow->ct = owner_idx;
13460                                 __atomic_fetch_add(&ct->refcnt, 1,
13461                                                    __ATOMIC_RELAXED);
13462                         }
13463                         actions_n++;
13464                         action_flags |= MLX5_FLOW_ACTION_CT;
13465                         break;
13466                 case RTE_FLOW_ACTION_TYPE_END:
13467                         actions_end = true;
13468                         if (mhdr_res->actions_num) {
13469                                 /* create modify action if needed. */
13470                                 if (flow_dv_modify_hdr_resource_register
13471                                         (dev, mhdr_res, dev_flow, error))
13472                                         return -rte_errno;
13473                                 dev_flow->dv.actions[modify_action_position] =
13474                                         handle->dvh.modify_hdr->action;
13475                         }
13476                         /*
13477                          * Handle AGE and COUNT action by single HW counter
13478                          * when they are not shared.
13479                          */
13480                         if (action_flags & MLX5_FLOW_ACTION_AGE) {
13481                                 if ((non_shared_age && count) ||
13482                                     !flow_hit_aso_supported(priv->sh, attr)) {
13483                                         /* Creates age by counters. */
13484                                         cnt_act = flow_dv_prepare_counter
13485                                                                 (dev, dev_flow,
13486                                                                  flow, count,
13487                                                                  non_shared_age,
13488                                                                  error);
13489                                         if (!cnt_act)
13490                                                 return -rte_errno;
13491                                         dev_flow->dv.actions[age_act_pos] =
13492                                                                 cnt_act->action;
13493                                         break;
13494                                 }
13495                                 if (!flow->age && non_shared_age) {
13496                                         flow->age = flow_dv_aso_age_alloc
13497                                                                 (dev, error);
13498                                         if (!flow->age)
13499                                                 return -rte_errno;
13500                                         flow_dv_aso_age_params_init
13501                                                     (dev, flow->age,
13502                                                      non_shared_age->context ?
13503                                                      non_shared_age->context :
13504                                                      (void *)(uintptr_t)
13505                                                      (dev_flow->flow_idx),
13506                                                      non_shared_age->timeout);
13507                                 }
13508                                 age_act = flow_aso_age_get_by_idx(dev,
13509                                                                   flow->age);
13510                                 dev_flow->dv.actions[age_act_pos] =
13511                                                              age_act->dr_action;
13512                         }
13513                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
13514                                 /*
13515                                  * Create one count action, to be used
13516                                  * by all sub-flows.
13517                                  */
13518                                 cnt_act = flow_dv_prepare_counter(dev, dev_flow,
13519                                                                   flow, count,
13520                                                                   NULL, error);
13521                                 if (!cnt_act)
13522                                         return -rte_errno;
13523                                 dev_flow->dv.actions[actions_n++] =
13524                                                                 cnt_act->action;
13525                         }
13526                 default:
13527                         break;
13528                 }
13529                 if (mhdr_res->actions_num &&
13530                     modify_action_position == UINT32_MAX)
13531                         modify_action_position = actions_n++;
13532         }
13533         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
13534                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
13535                 int item_type = items->type;
13536
13537                 if (!mlx5_flow_os_item_supported(item_type))
13538                         return rte_flow_error_set(error, ENOTSUP,
13539                                                   RTE_FLOW_ERROR_TYPE_ITEM,
13540                                                   NULL, "item not supported");
13541                 switch (item_type) {
13542                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
13543                         flow_dv_translate_item_port_id
13544                                 (dev, match_mask, match_value, items, attr);
13545                         last_item = MLX5_FLOW_ITEM_PORT_ID;
13546                         break;
13547                 case RTE_FLOW_ITEM_TYPE_ETH:
13548                         flow_dv_translate_item_eth(match_mask, match_value,
13549                                                    items, tunnel,
13550                                                    dev_flow->dv.group);
13551                         matcher.priority = action_flags &
13552                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
13553                                         !dev_flow->external ?
13554                                         MLX5_PRIORITY_MAP_L3 :
13555                                         MLX5_PRIORITY_MAP_L2;
13556                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
13557                                              MLX5_FLOW_LAYER_OUTER_L2;
13558                         break;
13559                 case RTE_FLOW_ITEM_TYPE_VLAN:
13560                         flow_dv_translate_item_vlan(dev_flow,
13561                                                     match_mask, match_value,
13562                                                     items, tunnel,
13563                                                     dev_flow->dv.group);
13564                         matcher.priority = MLX5_PRIORITY_MAP_L2;
13565                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
13566                                               MLX5_FLOW_LAYER_INNER_VLAN) :
13567                                              (MLX5_FLOW_LAYER_OUTER_L2 |
13568                                               MLX5_FLOW_LAYER_OUTER_VLAN);
13569                         break;
13570                 case RTE_FLOW_ITEM_TYPE_IPV4:
13571                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13572                                                   &item_flags, &tunnel);
13573                         flow_dv_translate_item_ipv4(match_mask, match_value,
13574                                                     items, tunnel,
13575                                                     dev_flow->dv.group);
13576                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13577                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
13578                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
13579                         if (items->mask != NULL &&
13580                             ((const struct rte_flow_item_ipv4 *)
13581                              items->mask)->hdr.next_proto_id) {
13582                                 next_protocol =
13583                                         ((const struct rte_flow_item_ipv4 *)
13584                                          (items->spec))->hdr.next_proto_id;
13585                                 next_protocol &=
13586                                         ((const struct rte_flow_item_ipv4 *)
13587                                          (items->mask))->hdr.next_proto_id;
13588                         } else {
13589                                 /* Reset for inner layer. */
13590                                 next_protocol = 0xff;
13591                         }
13592                         break;
13593                 case RTE_FLOW_ITEM_TYPE_IPV6:
13594                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13595                                                   &item_flags, &tunnel);
13596                         flow_dv_translate_item_ipv6(match_mask, match_value,
13597                                                     items, tunnel,
13598                                                     dev_flow->dv.group);
13599                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13600                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
13601                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
13602                         if (items->mask != NULL &&
13603                             ((const struct rte_flow_item_ipv6 *)
13604                              items->mask)->hdr.proto) {
13605                                 next_protocol =
13606                                         ((const struct rte_flow_item_ipv6 *)
13607                                          items->spec)->hdr.proto;
13608                                 next_protocol &=
13609                                         ((const struct rte_flow_item_ipv6 *)
13610                                          items->mask)->hdr.proto;
13611                         } else {
13612                                 /* Reset for inner layer. */
13613                                 next_protocol = 0xff;
13614                         }
13615                         break;
13616                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
13617                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
13618                                                              match_value,
13619                                                              items, tunnel);
13620                         last_item = tunnel ?
13621                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
13622                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
13623                         if (items->mask != NULL &&
13624                             ((const struct rte_flow_item_ipv6_frag_ext *)
13625                              items->mask)->hdr.next_header) {
13626                                 next_protocol =
13627                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13628                                  items->spec)->hdr.next_header;
13629                                 next_protocol &=
13630                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13631                                  items->mask)->hdr.next_header;
13632                         } else {
13633                                 /* Reset for inner layer. */
13634                                 next_protocol = 0xff;
13635                         }
13636                         break;
13637                 case RTE_FLOW_ITEM_TYPE_TCP:
13638                         flow_dv_translate_item_tcp(match_mask, match_value,
13639                                                    items, tunnel);
13640                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13641                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
13642                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
13643                         break;
13644                 case RTE_FLOW_ITEM_TYPE_UDP:
13645                         flow_dv_translate_item_udp(match_mask, match_value,
13646                                                    items, tunnel);
13647                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13648                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
13649                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
13650                         break;
13651                 case RTE_FLOW_ITEM_TYPE_GRE:
13652                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13653                         last_item = MLX5_FLOW_LAYER_GRE;
13654                         tunnel_item = items;
13655                         gre_item = items;
13656                         break;
13657                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
13658                         flow_dv_translate_item_gre_key(match_mask,
13659                                                        match_value, items);
13660                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
13661                         break;
13662                 case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
13663                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13664                         last_item = MLX5_FLOW_LAYER_GRE;
13665                         tunnel_item = items;
13666                         break;
13667                 case RTE_FLOW_ITEM_TYPE_NVGRE:
13668                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13669                         last_item = MLX5_FLOW_LAYER_GRE;
13670                         tunnel_item = items;
13671                         break;
13672                 case RTE_FLOW_ITEM_TYPE_VXLAN:
13673                         flow_dv_translate_item_vxlan(dev, attr,
13674                                                      match_mask, match_value,
13675                                                      items, tunnel);
13676                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13677                         last_item = MLX5_FLOW_LAYER_VXLAN;
13678                         break;
13679                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
13680                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13681                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
13682                         tunnel_item = items;
13683                         break;
13684                 case RTE_FLOW_ITEM_TYPE_GENEVE:
13685                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13686                         last_item = MLX5_FLOW_LAYER_GENEVE;
13687                         tunnel_item = items;
13688                         break;
13689                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
13690                         ret = flow_dv_translate_item_geneve_opt(dev, match_mask,
13691                                                           match_value,
13692                                                           items, error);
13693                         if (ret)
13694                                 return rte_flow_error_set(error, -ret,
13695                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13696                                         "cannot create GENEVE TLV option");
13697                         flow->geneve_tlv_option = 1;
13698                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
13699                         break;
13700                 case RTE_FLOW_ITEM_TYPE_MPLS:
13701                         flow_dv_translate_item_mpls(match_mask, match_value,
13702                                                     items, last_item, tunnel);
13703                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13704                         last_item = MLX5_FLOW_LAYER_MPLS;
13705                         break;
13706                 case RTE_FLOW_ITEM_TYPE_MARK:
13707                         flow_dv_translate_item_mark(dev, match_mask,
13708                                                     match_value, items);
13709                         last_item = MLX5_FLOW_ITEM_MARK;
13710                         break;
13711                 case RTE_FLOW_ITEM_TYPE_META:
13712                         flow_dv_translate_item_meta(dev, match_mask,
13713                                                     match_value, attr, items);
13714                         last_item = MLX5_FLOW_ITEM_METADATA;
13715                         break;
13716                 case RTE_FLOW_ITEM_TYPE_ICMP:
13717                         flow_dv_translate_item_icmp(match_mask, match_value,
13718                                                     items, tunnel);
13719                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13720                         last_item = MLX5_FLOW_LAYER_ICMP;
13721                         break;
13722                 case RTE_FLOW_ITEM_TYPE_ICMP6:
13723                         flow_dv_translate_item_icmp6(match_mask, match_value,
13724                                                       items, tunnel);
13725                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13726                         last_item = MLX5_FLOW_LAYER_ICMP6;
13727                         break;
13728                 case RTE_FLOW_ITEM_TYPE_TAG:
13729                         flow_dv_translate_item_tag(dev, match_mask,
13730                                                    match_value, items);
13731                         last_item = MLX5_FLOW_ITEM_TAG;
13732                         break;
13733                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
13734                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
13735                                                         match_value, items);
13736                         last_item = MLX5_FLOW_ITEM_TAG;
13737                         break;
13738                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
13739                         flow_dv_translate_item_tx_queue(dev, match_mask,
13740                                                         match_value,
13741                                                         items);
13742                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
13743                         break;
13744                 case RTE_FLOW_ITEM_TYPE_GTP:
13745                         flow_dv_translate_item_gtp(match_mask, match_value,
13746                                                    items, tunnel);
13747                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13748                         last_item = MLX5_FLOW_LAYER_GTP;
13749                         break;
13750                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
13751                         ret = flow_dv_translate_item_gtp_psc(match_mask,
13752                                                           match_value,
13753                                                           items);
13754                         if (ret)
13755                                 return rte_flow_error_set(error, -ret,
13756                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13757                                         "cannot create GTP PSC item");
13758                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
13759                         break;
13760                 case RTE_FLOW_ITEM_TYPE_ECPRI:
13761                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
13762                                 /* Create it only the first time to be used. */
13763                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
13764                                 if (ret)
13765                                         return rte_flow_error_set
13766                                                 (error, -ret,
13767                                                 RTE_FLOW_ERROR_TYPE_ITEM,
13768                                                 NULL,
13769                                                 "cannot create eCPRI parser");
13770                         }
13771                         flow_dv_translate_item_ecpri(dev, match_mask,
13772                                                      match_value, items,
13773                                                      last_item);
13774                         /* No other protocol should follow eCPRI layer. */
13775                         last_item = MLX5_FLOW_LAYER_ECPRI;
13776                         break;
13777                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
13778                         flow_dv_translate_item_integrity(items, integrity_items,
13779                                                          &last_item);
13780                         break;
13781                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
13782                         flow_dv_translate_item_aso_ct(dev, match_mask,
13783                                                       match_value, items);
13784                         break;
13785                 case RTE_FLOW_ITEM_TYPE_FLEX:
13786                         flow_dv_translate_item_flex(dev, match_mask,
13787                                                     match_value, items,
13788                                                     dev_flow, tunnel != 0);
13789                         last_item = tunnel ? MLX5_FLOW_ITEM_INNER_FLEX :
13790                                     MLX5_FLOW_ITEM_OUTER_FLEX;
13791                         break;
13792                 default:
13793                         break;
13794                 }
13795                 item_flags |= last_item;
13796         }
13797         /*
13798          * When E-Switch mode is enabled, we have two cases where we need to
13799          * set the source port manually.
13800          * The first one, is in case of NIC ingress steering rule, and the
13801          * second is E-Switch rule where no port_id item was found.
13802          * In both cases the source port is set according the current port
13803          * in use.
13804          */
13805         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && priv->sh->esw_mode &&
13806             !(attr->egress && !attr->transfer)) {
13807                 if (flow_dv_translate_item_port_id(dev, match_mask,
13808                                                    match_value, NULL, attr))
13809                         return -rte_errno;
13810         }
13811         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY) {
13812                 flow_dv_translate_item_integrity_post(match_mask, match_value,
13813                                                       integrity_items,
13814                                                       item_flags);
13815         }
13816         if (item_flags & MLX5_FLOW_LAYER_VXLAN_GPE)
13817                 flow_dv_translate_item_vxlan_gpe(match_mask, match_value,
13818                                                  tunnel_item, item_flags);
13819         else if (item_flags & MLX5_FLOW_LAYER_GENEVE)
13820                 flow_dv_translate_item_geneve(match_mask, match_value,
13821                                               tunnel_item, item_flags);
13822         else if (item_flags & MLX5_FLOW_LAYER_GRE) {
13823                 if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_GRE)
13824                         flow_dv_translate_item_gre(match_mask, match_value,
13825                                                    tunnel_item, item_flags);
13826                 else if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_NVGRE)
13827                         flow_dv_translate_item_nvgre(match_mask, match_value,
13828                                                      tunnel_item, item_flags);
13829                 else if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_GRE_OPTION)
13830                         flow_dv_translate_item_gre_option(match_mask, match_value,
13831                                         tunnel_item, gre_item, item_flags);
13832                 else
13833                         MLX5_ASSERT(false);
13834         }
13835 #ifdef RTE_LIBRTE_MLX5_DEBUG
13836         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
13837                                               dev_flow->dv.value.buf));
13838 #endif
13839         /*
13840          * Layers may be already initialized from prefix flow if this dev_flow
13841          * is the suffix flow.
13842          */
13843         handle->layers |= item_flags;
13844         if (action_flags & MLX5_FLOW_ACTION_RSS)
13845                 flow_dv_hashfields_set(dev_flow->handle->layers,
13846                                        rss_desc,
13847                                        &dev_flow->hash_fields);
13848         /* If has RSS action in the sample action, the Sample/Mirror resource
13849          * should be registered after the hash filed be update.
13850          */
13851         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
13852                 ret = flow_dv_translate_action_sample(dev,
13853                                                       sample,
13854                                                       dev_flow, attr,
13855                                                       &num_of_dest,
13856                                                       sample_actions,
13857                                                       &sample_res,
13858                                                       error);
13859                 if (ret < 0)
13860                         return ret;
13861                 ret = flow_dv_create_action_sample(dev,
13862                                                    dev_flow,
13863                                                    num_of_dest,
13864                                                    &sample_res,
13865                                                    &mdest_res,
13866                                                    sample_actions,
13867                                                    action_flags,
13868                                                    error);
13869                 if (ret < 0)
13870                         return rte_flow_error_set
13871                                                 (error, rte_errno,
13872                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13873                                                 NULL,
13874                                                 "cannot create sample action");
13875                 if (num_of_dest > 1) {
13876                         dev_flow->dv.actions[sample_act_pos] =
13877                         dev_flow->dv.dest_array_res->action;
13878                 } else {
13879                         dev_flow->dv.actions[sample_act_pos] =
13880                         dev_flow->dv.sample_res->verbs_action;
13881                 }
13882         }
13883         /*
13884          * For multiple destination (sample action with ratio=1), the encap
13885          * action and port id action will be combined into group action.
13886          * So need remove the original these actions in the flow and only
13887          * use the sample action instead of.
13888          */
13889         if (num_of_dest > 1 &&
13890             (sample_act->dr_port_id_action || sample_act->dr_jump_action)) {
13891                 int i;
13892                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
13893
13894                 for (i = 0; i < actions_n; i++) {
13895                         if ((sample_act->dr_encap_action &&
13896                                 sample_act->dr_encap_action ==
13897                                 dev_flow->dv.actions[i]) ||
13898                                 (sample_act->dr_port_id_action &&
13899                                 sample_act->dr_port_id_action ==
13900                                 dev_flow->dv.actions[i]) ||
13901                                 (sample_act->dr_jump_action &&
13902                                 sample_act->dr_jump_action ==
13903                                 dev_flow->dv.actions[i]))
13904                                 continue;
13905                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
13906                 }
13907                 memcpy((void *)dev_flow->dv.actions,
13908                                 (void *)temp_actions,
13909                                 tmp_actions_n * sizeof(void *));
13910                 actions_n = tmp_actions_n;
13911         }
13912         dev_flow->dv.actions_n = actions_n;
13913         dev_flow->act_flags = action_flags;
13914         if (wks->skip_matcher_reg)
13915                 return 0;
13916         /* Register matcher. */
13917         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
13918                                     matcher.mask.size);
13919         matcher.priority = mlx5_get_matcher_priority(dev, attr,
13920                                                      matcher.priority,
13921                                                      dev_flow->external);
13922         /**
13923          * When creating meter drop flow in drop table, using original
13924          * 5-tuple match, the matcher priority should be lower than
13925          * mtr_id matcher.
13926          */
13927         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
13928             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP &&
13929             matcher.priority <= MLX5_REG_BITS)
13930                 matcher.priority += MLX5_REG_BITS;
13931         /* reserved field no needs to be set to 0 here. */
13932         tbl_key.is_fdb = attr->transfer;
13933         tbl_key.is_egress = attr->egress;
13934         tbl_key.level = dev_flow->dv.group;
13935         tbl_key.id = dev_flow->dv.table_id;
13936         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
13937                                      tunnel, attr->group, error))
13938                 return -rte_errno;
13939         return 0;
13940 }
13941
13942 /**
13943  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13944  * and tunnel.
13945  *
13946  * @param[in, out] action
13947  *   Shred RSS action holding hash RX queue objects.
13948  * @param[in] hash_fields
13949  *   Defines combination of packet fields to participate in RX hash.
13950  * @param[in] tunnel
13951  *   Tunnel type
13952  * @param[in] hrxq_idx
13953  *   Hash RX queue index to set.
13954  *
13955  * @return
13956  *   0 on success, otherwise negative errno value.
13957  */
13958 static int
13959 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
13960                               const uint64_t hash_fields,
13961                               uint32_t hrxq_idx)
13962 {
13963         uint32_t *hrxqs = action->hrxq;
13964
13965         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13966         case MLX5_RSS_HASH_IPV4:
13967                 /* fall-through. */
13968         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13969                 /* fall-through. */
13970         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13971                 hrxqs[0] = hrxq_idx;
13972                 return 0;
13973         case MLX5_RSS_HASH_IPV4_TCP:
13974                 /* fall-through. */
13975         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13976                 /* fall-through. */
13977         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13978                 hrxqs[1] = hrxq_idx;
13979                 return 0;
13980         case MLX5_RSS_HASH_IPV4_UDP:
13981                 /* fall-through. */
13982         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13983                 /* fall-through. */
13984         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13985                 hrxqs[2] = hrxq_idx;
13986                 return 0;
13987         case MLX5_RSS_HASH_IPV6:
13988                 /* fall-through. */
13989         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13990                 /* fall-through. */
13991         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13992                 hrxqs[3] = hrxq_idx;
13993                 return 0;
13994         case MLX5_RSS_HASH_IPV6_TCP:
13995                 /* fall-through. */
13996         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13997                 /* fall-through. */
13998         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13999                 hrxqs[4] = hrxq_idx;
14000                 return 0;
14001         case MLX5_RSS_HASH_IPV6_UDP:
14002                 /* fall-through. */
14003         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
14004                 /* fall-through. */
14005         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
14006                 hrxqs[5] = hrxq_idx;
14007                 return 0;
14008         case MLX5_RSS_HASH_NONE:
14009                 hrxqs[6] = hrxq_idx;
14010                 return 0;
14011         default:
14012                 return -1;
14013         }
14014 }
14015
14016 /**
14017  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
14018  * and tunnel.
14019  *
14020  * @param[in] dev
14021  *   Pointer to the Ethernet device structure.
14022  * @param[in] idx
14023  *   Shared RSS action ID holding hash RX queue objects.
14024  * @param[in] hash_fields
14025  *   Defines combination of packet fields to participate in RX hash.
14026  * @param[in] tunnel
14027  *   Tunnel type
14028  *
14029  * @return
14030  *   Valid hash RX queue index, otherwise 0.
14031  */
14032 uint32_t
14033 flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
14034                                const uint64_t hash_fields)
14035 {
14036         struct mlx5_priv *priv = dev->data->dev_private;
14037         struct mlx5_shared_action_rss *shared_rss =
14038             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14039         const uint32_t *hrxqs = shared_rss->hrxq;
14040
14041         switch (hash_fields & ~IBV_RX_HASH_INNER) {
14042         case MLX5_RSS_HASH_IPV4:
14043                 /* fall-through. */
14044         case MLX5_RSS_HASH_IPV4_DST_ONLY:
14045                 /* fall-through. */
14046         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
14047                 return hrxqs[0];
14048         case MLX5_RSS_HASH_IPV4_TCP:
14049                 /* fall-through. */
14050         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
14051                 /* fall-through. */
14052         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
14053                 return hrxqs[1];
14054         case MLX5_RSS_HASH_IPV4_UDP:
14055                 /* fall-through. */
14056         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
14057                 /* fall-through. */
14058         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
14059                 return hrxqs[2];
14060         case MLX5_RSS_HASH_IPV6:
14061                 /* fall-through. */
14062         case MLX5_RSS_HASH_IPV6_DST_ONLY:
14063                 /* fall-through. */
14064         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
14065                 return hrxqs[3];
14066         case MLX5_RSS_HASH_IPV6_TCP:
14067                 /* fall-through. */
14068         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
14069                 /* fall-through. */
14070         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
14071                 return hrxqs[4];
14072         case MLX5_RSS_HASH_IPV6_UDP:
14073                 /* fall-through. */
14074         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
14075                 /* fall-through. */
14076         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
14077                 return hrxqs[5];
14078         case MLX5_RSS_HASH_NONE:
14079                 return hrxqs[6];
14080         default:
14081                 return 0;
14082         }
14083
14084 }
14085
14086 /**
14087  * Apply the flow to the NIC, lock free,
14088  * (mutex should be acquired by caller).
14089  *
14090  * @param[in] dev
14091  *   Pointer to the Ethernet device structure.
14092  * @param[in, out] flow
14093  *   Pointer to flow structure.
14094  * @param[out] error
14095  *   Pointer to error structure.
14096  *
14097  * @return
14098  *   0 on success, a negative errno value otherwise and rte_errno is set.
14099  */
14100 static int
14101 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
14102               struct rte_flow_error *error)
14103 {
14104         struct mlx5_flow_dv_workspace *dv;
14105         struct mlx5_flow_handle *dh;
14106         struct mlx5_flow_handle_dv *dv_h;
14107         struct mlx5_flow *dev_flow;
14108         struct mlx5_priv *priv = dev->data->dev_private;
14109         uint32_t handle_idx;
14110         int n;
14111         int err;
14112         int idx;
14113         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
14114         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
14115         uint8_t misc_mask;
14116
14117         MLX5_ASSERT(wks);
14118         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
14119                 dev_flow = &wks->flows[idx];
14120                 dv = &dev_flow->dv;
14121                 dh = dev_flow->handle;
14122                 dv_h = &dh->dvh;
14123                 n = dv->actions_n;
14124                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
14125                         if (dv->transfer) {
14126                                 MLX5_ASSERT(priv->sh->dr_drop_action);
14127                                 dv->actions[n++] = priv->sh->dr_drop_action;
14128                         } else {
14129 #ifdef HAVE_MLX5DV_DR
14130                                 /* DR supports drop action placeholder. */
14131                                 MLX5_ASSERT(priv->sh->dr_drop_action);
14132                                 dv->actions[n++] = dv->group ?
14133                                         priv->sh->dr_drop_action :
14134                                         priv->root_drop_action;
14135 #else
14136                                 /* For DV we use the explicit drop queue. */
14137                                 MLX5_ASSERT(priv->drop_queue.hrxq);
14138                                 dv->actions[n++] =
14139                                                 priv->drop_queue.hrxq->action;
14140 #endif
14141                         }
14142                 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
14143                            !dv_h->rix_sample && !dv_h->rix_dest_array)) {
14144                         struct mlx5_hrxq *hrxq;
14145                         uint32_t hrxq_idx;
14146
14147                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc,
14148                                                     &hrxq_idx);
14149                         if (!hrxq) {
14150                                 rte_flow_error_set
14151                                         (error, rte_errno,
14152                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14153                                          "cannot get hash queue");
14154                                 goto error;
14155                         }
14156                         dh->rix_hrxq = hrxq_idx;
14157                         dv->actions[n++] = hrxq->action;
14158                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
14159                         struct mlx5_hrxq *hrxq = NULL;
14160                         uint32_t hrxq_idx;
14161
14162                         hrxq_idx = flow_dv_action_rss_hrxq_lookup(dev,
14163                                                 rss_desc->shared_rss,
14164                                                 dev_flow->hash_fields);
14165                         if (hrxq_idx)
14166                                 hrxq = mlx5_ipool_get
14167                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
14168                                          hrxq_idx);
14169                         if (!hrxq) {
14170                                 rte_flow_error_set
14171                                         (error, rte_errno,
14172                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14173                                          "cannot get hash queue");
14174                                 goto error;
14175                         }
14176                         dh->rix_srss = rss_desc->shared_rss;
14177                         dv->actions[n++] = hrxq->action;
14178                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
14179                         if (!priv->sh->default_miss_action) {
14180                                 rte_flow_error_set
14181                                         (error, rte_errno,
14182                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14183                                          "default miss action not be created.");
14184                                 goto error;
14185                         }
14186                         dv->actions[n++] = priv->sh->default_miss_action;
14187                 }
14188                 misc_mask = flow_dv_matcher_enable(dv->value.buf);
14189                 __flow_dv_adjust_buf_size(&dv->value.size, misc_mask);
14190                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
14191                                                (void *)&dv->value, n,
14192                                                dv->actions, &dh->drv_flow);
14193                 if (err) {
14194                         rte_flow_error_set
14195                                 (error, errno,
14196                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14197                                 NULL,
14198                                 (!priv->sh->config.allow_duplicate_pattern &&
14199                                 errno == EEXIST) ?
14200                                 "duplicating pattern is not allowed" :
14201                                 "hardware refuses to create flow");
14202                         goto error;
14203                 }
14204                 if (priv->vmwa_context &&
14205                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
14206                         /*
14207                          * The rule contains the VLAN pattern.
14208                          * For VF we are going to create VLAN
14209                          * interface to make hypervisor set correct
14210                          * e-Switch vport context.
14211                          */
14212                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
14213                 }
14214         }
14215         return 0;
14216 error:
14217         err = rte_errno; /* Save rte_errno before cleanup. */
14218         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
14219                        handle_idx, dh, next) {
14220                 /* hrxq is union, don't clear it if the flag is not set. */
14221                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
14222                         mlx5_hrxq_release(dev, dh->rix_hrxq);
14223                         dh->rix_hrxq = 0;
14224                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
14225                         dh->rix_srss = 0;
14226                 }
14227                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
14228                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
14229         }
14230         rte_errno = err; /* Restore rte_errno. */
14231         return -rte_errno;
14232 }
14233
14234 void
14235 flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
14236                           struct mlx5_list_entry *entry)
14237 {
14238         struct mlx5_flow_dv_matcher *resource = container_of(entry,
14239                                                              typeof(*resource),
14240                                                              entry);
14241
14242         claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
14243         mlx5_free(resource);
14244 }
14245
14246 /**
14247  * Release the flow matcher.
14248  *
14249  * @param dev
14250  *   Pointer to Ethernet device.
14251  * @param port_id
14252  *   Index to port ID action resource.
14253  *
14254  * @return
14255  *   1 while a reference on it exists, 0 when freed.
14256  */
14257 static int
14258 flow_dv_matcher_release(struct rte_eth_dev *dev,
14259                         struct mlx5_flow_handle *handle)
14260 {
14261         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
14262         struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl,
14263                                                             typeof(*tbl), tbl);
14264         int ret;
14265
14266         MLX5_ASSERT(matcher->matcher_object);
14267         ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
14268         flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
14269         return ret;
14270 }
14271
14272 void
14273 flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14274 {
14275         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14276         struct mlx5_flow_dv_encap_decap_resource *res =
14277                                        container_of(entry, typeof(*res), entry);
14278
14279         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
14280         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
14281 }
14282
14283 /**
14284  * Release an encap/decap resource.
14285  *
14286  * @param dev
14287  *   Pointer to Ethernet device.
14288  * @param encap_decap_idx
14289  *   Index of encap decap resource.
14290  *
14291  * @return
14292  *   1 while a reference on it exists, 0 when freed.
14293  */
14294 static int
14295 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
14296                                      uint32_t encap_decap_idx)
14297 {
14298         struct mlx5_priv *priv = dev->data->dev_private;
14299         struct mlx5_flow_dv_encap_decap_resource *resource;
14300
14301         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
14302                                   encap_decap_idx);
14303         if (!resource)
14304                 return 0;
14305         MLX5_ASSERT(resource->action);
14306         return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
14307 }
14308
14309 /**
14310  * Release an jump to table action resource.
14311  *
14312  * @param dev
14313  *   Pointer to Ethernet device.
14314  * @param rix_jump
14315  *   Index to the jump action resource.
14316  *
14317  * @return
14318  *   1 while a reference on it exists, 0 when freed.
14319  */
14320 static int
14321 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
14322                                   uint32_t rix_jump)
14323 {
14324         struct mlx5_priv *priv = dev->data->dev_private;
14325         struct mlx5_flow_tbl_data_entry *tbl_data;
14326
14327         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
14328                                   rix_jump);
14329         if (!tbl_data)
14330                 return 0;
14331         return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl);
14332 }
14333
14334 void
14335 flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14336 {
14337         struct mlx5_flow_dv_modify_hdr_resource *res =
14338                 container_of(entry, typeof(*res), entry);
14339         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14340
14341         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
14342         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
14343 }
14344
14345 /**
14346  * Release a modify-header resource.
14347  *
14348  * @param dev
14349  *   Pointer to Ethernet device.
14350  * @param handle
14351  *   Pointer to mlx5_flow_handle.
14352  *
14353  * @return
14354  *   1 while a reference on it exists, 0 when freed.
14355  */
14356 static int
14357 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
14358                                     struct mlx5_flow_handle *handle)
14359 {
14360         struct mlx5_priv *priv = dev->data->dev_private;
14361         struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr;
14362
14363         MLX5_ASSERT(entry->action);
14364         return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry);
14365 }
14366
14367 void
14368 flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14369 {
14370         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14371         struct mlx5_flow_dv_port_id_action_resource *resource =
14372                                   container_of(entry, typeof(*resource), entry);
14373
14374         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14375         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
14376 }
14377
14378 /**
14379  * Release port ID action resource.
14380  *
14381  * @param dev
14382  *   Pointer to Ethernet device.
14383  * @param handle
14384  *   Pointer to mlx5_flow_handle.
14385  *
14386  * @return
14387  *   1 while a reference on it exists, 0 when freed.
14388  */
14389 static int
14390 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
14391                                         uint32_t port_id)
14392 {
14393         struct mlx5_priv *priv = dev->data->dev_private;
14394         struct mlx5_flow_dv_port_id_action_resource *resource;
14395
14396         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
14397         if (!resource)
14398                 return 0;
14399         MLX5_ASSERT(resource->action);
14400         return mlx5_list_unregister(priv->sh->port_id_action_list,
14401                                     &resource->entry);
14402 }
14403
14404 /**
14405  * Release shared RSS action resource.
14406  *
14407  * @param dev
14408  *   Pointer to Ethernet device.
14409  * @param srss
14410  *   Shared RSS action index.
14411  */
14412 static void
14413 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
14414 {
14415         struct mlx5_priv *priv = dev->data->dev_private;
14416         struct mlx5_shared_action_rss *shared_rss;
14417
14418         shared_rss = mlx5_ipool_get
14419                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss);
14420         __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14421 }
14422
14423 void
14424 flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14425 {
14426         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14427         struct mlx5_flow_dv_push_vlan_action_resource *resource =
14428                         container_of(entry, typeof(*resource), entry);
14429
14430         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14431         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
14432 }
14433
14434 /**
14435  * Release push vlan action resource.
14436  *
14437  * @param dev
14438  *   Pointer to Ethernet device.
14439  * @param handle
14440  *   Pointer to mlx5_flow_handle.
14441  *
14442  * @return
14443  *   1 while a reference on it exists, 0 when freed.
14444  */
14445 static int
14446 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
14447                                           struct mlx5_flow_handle *handle)
14448 {
14449         struct mlx5_priv *priv = dev->data->dev_private;
14450         struct mlx5_flow_dv_push_vlan_action_resource *resource;
14451         uint32_t idx = handle->dvh.rix_push_vlan;
14452
14453         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
14454         if (!resource)
14455                 return 0;
14456         MLX5_ASSERT(resource->action);
14457         return mlx5_list_unregister(priv->sh->push_vlan_action_list,
14458                                     &resource->entry);
14459 }
14460
14461 /**
14462  * Release the fate resource.
14463  *
14464  * @param dev
14465  *   Pointer to Ethernet device.
14466  * @param handle
14467  *   Pointer to mlx5_flow_handle.
14468  */
14469 static void
14470 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
14471                                struct mlx5_flow_handle *handle)
14472 {
14473         if (!handle->rix_fate)
14474                 return;
14475         switch (handle->fate_action) {
14476         case MLX5_FLOW_FATE_QUEUE:
14477                 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array)
14478                         mlx5_hrxq_release(dev, handle->rix_hrxq);
14479                 break;
14480         case MLX5_FLOW_FATE_JUMP:
14481                 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump);
14482                 break;
14483         case MLX5_FLOW_FATE_PORT_ID:
14484                 flow_dv_port_id_action_resource_release(dev,
14485                                 handle->rix_port_id_action);
14486                 break;
14487         default:
14488                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
14489                 break;
14490         }
14491         handle->rix_fate = 0;
14492 }
14493
14494 void
14495 flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
14496                          struct mlx5_list_entry *entry)
14497 {
14498         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
14499                                                               typeof(*resource),
14500                                                               entry);
14501         struct rte_eth_dev *dev = resource->dev;
14502         struct mlx5_priv *priv = dev->data->dev_private;
14503
14504         if (resource->verbs_action)
14505                 claim_zero(mlx5_flow_os_destroy_flow_action
14506                                                       (resource->verbs_action));
14507         if (resource->normal_path_tbl)
14508                 flow_dv_tbl_resource_release(MLX5_SH(dev),
14509                                              resource->normal_path_tbl);
14510         flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
14511         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
14512         DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
14513 }
14514
14515 /**
14516  * Release an sample resource.
14517  *
14518  * @param dev
14519  *   Pointer to Ethernet device.
14520  * @param handle
14521  *   Pointer to mlx5_flow_handle.
14522  *
14523  * @return
14524  *   1 while a reference on it exists, 0 when freed.
14525  */
14526 static int
14527 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
14528                                      struct mlx5_flow_handle *handle)
14529 {
14530         struct mlx5_priv *priv = dev->data->dev_private;
14531         struct mlx5_flow_dv_sample_resource *resource;
14532
14533         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
14534                                   handle->dvh.rix_sample);
14535         if (!resource)
14536                 return 0;
14537         MLX5_ASSERT(resource->verbs_action);
14538         return mlx5_list_unregister(priv->sh->sample_action_list,
14539                                     &resource->entry);
14540 }
14541
14542 void
14543 flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
14544                              struct mlx5_list_entry *entry)
14545 {
14546         struct mlx5_flow_dv_dest_array_resource *resource =
14547                         container_of(entry, typeof(*resource), entry);
14548         struct rte_eth_dev *dev = resource->dev;
14549         struct mlx5_priv *priv = dev->data->dev_private;
14550         uint32_t i = 0;
14551
14552         MLX5_ASSERT(resource->action);
14553         if (resource->action)
14554                 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14555         for (; i < resource->num_of_dest; i++)
14556                 flow_dv_sample_sub_actions_release(dev,
14557                                                    &resource->sample_idx[i]);
14558         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
14559         DRV_LOG(DEBUG, "destination array resource %p: removed",
14560                 (void *)resource);
14561 }
14562
14563 /**
14564  * Release an destination array resource.
14565  *
14566  * @param dev
14567  *   Pointer to Ethernet device.
14568  * @param handle
14569  *   Pointer to mlx5_flow_handle.
14570  *
14571  * @return
14572  *   1 while a reference on it exists, 0 when freed.
14573  */
14574 static int
14575 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
14576                                     struct mlx5_flow_handle *handle)
14577 {
14578         struct mlx5_priv *priv = dev->data->dev_private;
14579         struct mlx5_flow_dv_dest_array_resource *resource;
14580
14581         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
14582                                   handle->dvh.rix_dest_array);
14583         if (!resource)
14584                 return 0;
14585         MLX5_ASSERT(resource->action);
14586         return mlx5_list_unregister(priv->sh->dest_array_list,
14587                                     &resource->entry);
14588 }
14589
14590 static void
14591 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev)
14592 {
14593         struct mlx5_priv *priv = dev->data->dev_private;
14594         struct mlx5_dev_ctx_shared *sh = priv->sh;
14595         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
14596                                 sh->geneve_tlv_option_resource;
14597         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
14598         if (geneve_opt_resource) {
14599                 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1,
14600                                          __ATOMIC_RELAXED))) {
14601                         claim_zero(mlx5_devx_cmd_destroy
14602                                         (geneve_opt_resource->obj));
14603                         mlx5_free(sh->geneve_tlv_option_resource);
14604                         sh->geneve_tlv_option_resource = NULL;
14605                 }
14606         }
14607         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
14608 }
14609
14610 /**
14611  * Remove the flow from the NIC but keeps it in memory.
14612  * Lock free, (mutex should be acquired by caller).
14613  *
14614  * @param[in] dev
14615  *   Pointer to Ethernet device.
14616  * @param[in, out] flow
14617  *   Pointer to flow structure.
14618  */
14619 static void
14620 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
14621 {
14622         struct mlx5_flow_handle *dh;
14623         uint32_t handle_idx;
14624         struct mlx5_priv *priv = dev->data->dev_private;
14625
14626         if (!flow)
14627                 return;
14628         handle_idx = flow->dev_handles;
14629         while (handle_idx) {
14630                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14631                                     handle_idx);
14632                 if (!dh)
14633                         return;
14634                 if (dh->drv_flow) {
14635                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
14636                         dh->drv_flow = NULL;
14637                 }
14638                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
14639                         flow_dv_fate_resource_release(dev, dh);
14640                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
14641                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
14642                 handle_idx = dh->next.next;
14643         }
14644 }
14645
14646 /**
14647  * Remove the flow from the NIC and the memory.
14648  * Lock free, (mutex should be acquired by caller).
14649  *
14650  * @param[in] dev
14651  *   Pointer to the Ethernet device structure.
14652  * @param[in, out] flow
14653  *   Pointer to flow structure.
14654  */
14655 static void
14656 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
14657 {
14658         struct mlx5_flow_handle *dev_handle;
14659         struct mlx5_priv *priv = dev->data->dev_private;
14660         struct mlx5_flow_meter_info *fm = NULL;
14661         uint32_t srss = 0;
14662
14663         if (!flow)
14664                 return;
14665         flow_dv_remove(dev, flow);
14666         if (flow->counter) {
14667                 flow_dv_counter_free(dev, flow->counter);
14668                 flow->counter = 0;
14669         }
14670         if (flow->meter) {
14671                 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
14672                 if (fm)
14673                         mlx5_flow_meter_detach(priv, fm);
14674                 flow->meter = 0;
14675         }
14676         /* Keep the current age handling by default. */
14677         if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct)
14678                 flow_dv_aso_ct_release(dev, flow->ct, NULL);
14679         else if (flow->age)
14680                 flow_dv_aso_age_release(dev, flow->age);
14681         if (flow->geneve_tlv_option) {
14682                 flow_dv_geneve_tlv_option_resource_release(dev);
14683                 flow->geneve_tlv_option = 0;
14684         }
14685         while (flow->dev_handles) {
14686                 uint32_t tmp_idx = flow->dev_handles;
14687
14688                 dev_handle = mlx5_ipool_get(priv->sh->ipool
14689                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
14690                 if (!dev_handle)
14691                         return;
14692                 flow->dev_handles = dev_handle->next.next;
14693                 while (dev_handle->flex_item) {
14694                         int index = rte_bsf32(dev_handle->flex_item);
14695
14696                         mlx5_flex_release_index(dev, index);
14697                         dev_handle->flex_item &= ~(uint8_t)RTE_BIT32(index);
14698                 }
14699                 if (dev_handle->dvh.matcher)
14700                         flow_dv_matcher_release(dev, dev_handle);
14701                 if (dev_handle->dvh.rix_sample)
14702                         flow_dv_sample_resource_release(dev, dev_handle);
14703                 if (dev_handle->dvh.rix_dest_array)
14704                         flow_dv_dest_array_resource_release(dev, dev_handle);
14705                 if (dev_handle->dvh.rix_encap_decap)
14706                         flow_dv_encap_decap_resource_release(dev,
14707                                 dev_handle->dvh.rix_encap_decap);
14708                 if (dev_handle->dvh.modify_hdr)
14709                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
14710                 if (dev_handle->dvh.rix_push_vlan)
14711                         flow_dv_push_vlan_action_resource_release(dev,
14712                                                                   dev_handle);
14713                 if (dev_handle->dvh.rix_tag)
14714                         flow_dv_tag_release(dev,
14715                                             dev_handle->dvh.rix_tag);
14716                 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS)
14717                         flow_dv_fate_resource_release(dev, dev_handle);
14718                 else if (!srss)
14719                         srss = dev_handle->rix_srss;
14720                 if (fm && dev_handle->is_meter_flow_id &&
14721                     dev_handle->split_flow_id)
14722                         mlx5_ipool_free(fm->flow_ipool,
14723                                         dev_handle->split_flow_id);
14724                 else if (dev_handle->split_flow_id &&
14725                     !dev_handle->is_meter_flow_id)
14726                         mlx5_ipool_free(priv->sh->ipool
14727                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
14728                                         dev_handle->split_flow_id);
14729                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14730                            tmp_idx);
14731         }
14732         if (srss)
14733                 flow_dv_shared_rss_action_release(dev, srss);
14734 }
14735
14736 /**
14737  * Release array of hash RX queue objects.
14738  * Helper function.
14739  *
14740  * @param[in] dev
14741  *   Pointer to the Ethernet device structure.
14742  * @param[in, out] hrxqs
14743  *   Array of hash RX queue objects.
14744  *
14745  * @return
14746  *   Total number of references to hash RX queue objects in *hrxqs* array
14747  *   after this operation.
14748  */
14749 static int
14750 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
14751                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
14752 {
14753         size_t i;
14754         int remaining = 0;
14755
14756         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
14757                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
14758
14759                 if (!ret)
14760                         (*hrxqs)[i] = 0;
14761                 remaining += ret;
14762         }
14763         return remaining;
14764 }
14765
14766 /**
14767  * Release all hash RX queue objects representing shared RSS action.
14768  *
14769  * @param[in] dev
14770  *   Pointer to the Ethernet device structure.
14771  * @param[in, out] action
14772  *   Shared RSS action to remove hash RX queue objects from.
14773  *
14774  * @return
14775  *   Total number of references to hash RX queue objects stored in *action*
14776  *   after this operation.
14777  *   Expected to be 0 if no external references held.
14778  */
14779 static int
14780 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
14781                                  struct mlx5_shared_action_rss *shared_rss)
14782 {
14783         return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq);
14784 }
14785
14786 /**
14787  * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to
14788  * user input.
14789  *
14790  * Only one hash value is available for one L3+L4 combination:
14791  * for example:
14792  * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and
14793  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
14794  * same slot in mlx5_rss_hash_fields.
14795  *
14796  * @param[in] orig_rss_types
14797  *   RSS type as provided in shared RSS action.
14798  * @param[in, out] hash_field
14799  *   hash_field variable needed to be adjusted.
14800  *
14801  * @return
14802  *   void
14803  */
14804 void
14805 flow_dv_action_rss_l34_hash_adjust(uint64_t orig_rss_types,
14806                                    uint64_t *hash_field)
14807 {
14808         uint64_t rss_types = rte_eth_rss_hf_refine(orig_rss_types);
14809
14810         switch (*hash_field & ~IBV_RX_HASH_INNER) {
14811         case MLX5_RSS_HASH_IPV4:
14812                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
14813                         *hash_field &= ~MLX5_RSS_HASH_IPV4;
14814                         if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
14815                                 *hash_field |= IBV_RX_HASH_DST_IPV4;
14816                         else if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
14817                                 *hash_field |= IBV_RX_HASH_SRC_IPV4;
14818                         else
14819                                 *hash_field |= MLX5_RSS_HASH_IPV4;
14820                 }
14821                 return;
14822         case MLX5_RSS_HASH_IPV6:
14823                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
14824                         *hash_field &= ~MLX5_RSS_HASH_IPV6;
14825                         if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
14826                                 *hash_field |= IBV_RX_HASH_DST_IPV6;
14827                         else if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
14828                                 *hash_field |= IBV_RX_HASH_SRC_IPV6;
14829                         else
14830                                 *hash_field |= MLX5_RSS_HASH_IPV6;
14831                 }
14832                 return;
14833         case MLX5_RSS_HASH_IPV4_UDP:
14834                 /* fall-through. */
14835         case MLX5_RSS_HASH_IPV6_UDP:
14836                 if (rss_types & RTE_ETH_RSS_UDP) {
14837                         *hash_field &= ~MLX5_UDP_IBV_RX_HASH;
14838                         if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
14839                                 *hash_field |= IBV_RX_HASH_DST_PORT_UDP;
14840                         else if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
14841                                 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP;
14842                         else
14843                                 *hash_field |= MLX5_UDP_IBV_RX_HASH;
14844                 }
14845                 return;
14846         case MLX5_RSS_HASH_IPV4_TCP:
14847                 /* fall-through. */
14848         case MLX5_RSS_HASH_IPV6_TCP:
14849                 if (rss_types & RTE_ETH_RSS_TCP) {
14850                         *hash_field &= ~MLX5_TCP_IBV_RX_HASH;
14851                         if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
14852                                 *hash_field |= IBV_RX_HASH_DST_PORT_TCP;
14853                         else if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
14854                                 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP;
14855                         else
14856                                 *hash_field |= MLX5_TCP_IBV_RX_HASH;
14857                 }
14858                 return;
14859         default:
14860                 return;
14861         }
14862 }
14863
14864 /**
14865  * Setup shared RSS action.
14866  * Prepare set of hash RX queue objects sufficient to handle all valid
14867  * hash_fields combinations (see enum ibv_rx_hash_fields).
14868  *
14869  * @param[in] dev
14870  *   Pointer to the Ethernet device structure.
14871  * @param[in] action_idx
14872  *   Shared RSS action ipool index.
14873  * @param[in, out] action
14874  *   Partially initialized shared RSS action.
14875  * @param[out] error
14876  *   Perform verbose error reporting if not NULL. Initialized in case of
14877  *   error only.
14878  *
14879  * @return
14880  *   0 on success, otherwise negative errno value.
14881  */
14882 static int
14883 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
14884                            uint32_t action_idx,
14885                            struct mlx5_shared_action_rss *shared_rss,
14886                            struct rte_flow_error *error)
14887 {
14888         struct mlx5_priv *priv = dev->data->dev_private;
14889         struct mlx5_flow_rss_desc rss_desc = { 0 };
14890         size_t i;
14891         int err;
14892
14893         shared_rss->ind_tbl = mlx5_ind_table_obj_new
14894                               (dev, shared_rss->origin.queue,
14895                                shared_rss->origin.queue_num,
14896                                true,
14897                                !!dev->data->dev_started);
14898         if (!shared_rss->ind_tbl)
14899                 return rte_flow_error_set(error, rte_errno,
14900                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14901                                           "cannot setup indirection table");
14902         memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN);
14903         rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
14904         rss_desc.const_q = shared_rss->origin.queue;
14905         rss_desc.queue_num = shared_rss->origin.queue_num;
14906         /* Set non-zero value to indicate a shared RSS. */
14907         rss_desc.shared_rss = action_idx;
14908         rss_desc.ind_tbl = shared_rss->ind_tbl;
14909         if (priv->sh->config.dv_flow_en == 2)
14910                 rss_desc.hws_flags = MLX5DR_ACTION_FLAG_HWS_RX;
14911         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
14912                 struct mlx5_hrxq *hrxq;
14913                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
14914                 int tunnel = 0;
14915
14916                 flow_dv_action_rss_l34_hash_adjust(shared_rss->origin.types,
14917                                                    &hash_fields);
14918                 if (shared_rss->origin.level > 1) {
14919                         hash_fields |= IBV_RX_HASH_INNER;
14920                         tunnel = 1;
14921                 }
14922                 rss_desc.tunnel = tunnel;
14923                 rss_desc.hash_fields = hash_fields;
14924                 hrxq = mlx5_hrxq_get(dev, &rss_desc);
14925                 if (!hrxq) {
14926                         rte_flow_error_set
14927                                 (error, rte_errno,
14928                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14929                                  "cannot get hash queue");
14930                         goto error_hrxq_new;
14931                 }
14932                 err = __flow_dv_action_rss_hrxq_set
14933                         (shared_rss, hash_fields, hrxq->idx);
14934                 MLX5_ASSERT(!err);
14935         }
14936         return 0;
14937 error_hrxq_new:
14938         err = rte_errno;
14939         __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14940         if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true))
14941                 shared_rss->ind_tbl = NULL;
14942         rte_errno = err;
14943         return -rte_errno;
14944 }
14945
14946 /**
14947  * Create shared RSS action.
14948  *
14949  * @param[in] dev
14950  *   Pointer to the Ethernet device structure.
14951  * @param[in] conf
14952  *   Shared action configuration.
14953  * @param[in] rss
14954  *   RSS action specification used to create shared action.
14955  * @param[out] error
14956  *   Perform verbose error reporting if not NULL. Initialized in case of
14957  *   error only.
14958  *
14959  * @return
14960  *   A valid shared action ID in case of success, 0 otherwise and
14961  *   rte_errno is set.
14962  */
14963 static uint32_t
14964 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
14965                             const struct rte_flow_indir_action_conf *conf,
14966                             const struct rte_flow_action_rss *rss,
14967                             struct rte_flow_error *error)
14968 {
14969         struct mlx5_priv *priv = dev->data->dev_private;
14970         struct mlx5_shared_action_rss *shared_rss = NULL;
14971         struct rte_flow_action_rss *origin;
14972         const uint8_t *rss_key;
14973         uint32_t idx;
14974
14975         RTE_SET_USED(conf);
14976         shared_rss = mlx5_ipool_zmalloc
14977                          (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx);
14978         if (!shared_rss) {
14979                 rte_flow_error_set(error, ENOMEM,
14980                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14981                                    "cannot allocate resource memory");
14982                 goto error_rss_init;
14983         }
14984         if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) {
14985                 rte_flow_error_set(error, E2BIG,
14986                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14987                                    "rss action number out of range");
14988                 goto error_rss_init;
14989         }
14990         origin = &shared_rss->origin;
14991         origin->func = rss->func;
14992         origin->level = rss->level;
14993         /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */
14994         origin->types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
14995         /* NULL RSS key indicates default RSS key. */
14996         rss_key = !rss->key ? rss_hash_default_key : rss->key;
14997         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
14998         origin->key = &shared_rss->key[0];
14999         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
15000         origin->queue = rss->queue;
15001         origin->queue_num = rss->queue_num;
15002         if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error))
15003                 goto error_rss_init;
15004         /* Update queue with indirect table queue memoyr. */
15005         origin->queue = shared_rss->ind_tbl->queues;
15006         rte_spinlock_init(&shared_rss->action_rss_sl);
15007         __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
15008         rte_spinlock_lock(&priv->shared_act_sl);
15009         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
15010                      &priv->rss_shared_actions, idx, shared_rss, next);
15011         rte_spinlock_unlock(&priv->shared_act_sl);
15012         return idx;
15013 error_rss_init:
15014         if (shared_rss) {
15015                 if (shared_rss->ind_tbl)
15016                         mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl,
15017                                                    !!dev->data->dev_started);
15018                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
15019                                 idx);
15020         }
15021         return 0;
15022 }
15023
15024 /**
15025  * Destroy the shared RSS action.
15026  * Release related hash RX queue objects.
15027  *
15028  * @param[in] dev
15029  *   Pointer to the Ethernet device structure.
15030  * @param[in] idx
15031  *   The shared RSS action object ID to be removed.
15032  * @param[out] error
15033  *   Perform verbose error reporting if not NULL. Initialized in case of
15034  *   error only.
15035  *
15036  * @return
15037  *   0 on success, otherwise negative errno value.
15038  */
15039 static int
15040 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx,
15041                              struct rte_flow_error *error)
15042 {
15043         struct mlx5_priv *priv = dev->data->dev_private;
15044         struct mlx5_shared_action_rss *shared_rss =
15045             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
15046         uint32_t old_refcnt = 1;
15047         int remaining;
15048
15049         if (!shared_rss)
15050                 return rte_flow_error_set(error, EINVAL,
15051                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15052                                           "invalid shared action");
15053         if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt,
15054                                          0, 0, __ATOMIC_ACQUIRE,
15055                                          __ATOMIC_RELAXED))
15056                 return rte_flow_error_set(error, EBUSY,
15057                                           RTE_FLOW_ERROR_TYPE_ACTION,
15058                                           NULL,
15059                                           "shared rss has references");
15060         remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
15061         if (remaining)
15062                 return rte_flow_error_set(error, EBUSY,
15063                                           RTE_FLOW_ERROR_TYPE_ACTION,
15064                                           NULL,
15065                                           "shared rss hrxq has references");
15066         remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl,
15067                                                !!dev->data->dev_started);
15068         if (remaining)
15069                 return rte_flow_error_set(error, EBUSY,
15070                                           RTE_FLOW_ERROR_TYPE_ACTION,
15071                                           NULL,
15072                                           "shared rss indirection table has"
15073                                           " references");
15074         rte_spinlock_lock(&priv->shared_act_sl);
15075         ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
15076                      &priv->rss_shared_actions, idx, shared_rss, next);
15077         rte_spinlock_unlock(&priv->shared_act_sl);
15078         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
15079                         idx);
15080         return 0;
15081 }
15082
15083 /**
15084  * Create indirect action, lock free,
15085  * (mutex should be acquired by caller).
15086  * Dispatcher for action type specific call.
15087  *
15088  * @param[in] dev
15089  *   Pointer to the Ethernet device structure.
15090  * @param[in] conf
15091  *   Shared action configuration.
15092  * @param[in] action
15093  *   Action specification used to create indirect action.
15094  * @param[out] error
15095  *   Perform verbose error reporting if not NULL. Initialized in case of
15096  *   error only.
15097  *
15098  * @return
15099  *   A valid shared action handle in case of success, NULL otherwise and
15100  *   rte_errno is set.
15101  */
15102 struct rte_flow_action_handle *
15103 flow_dv_action_create(struct rte_eth_dev *dev,
15104                       const struct rte_flow_indir_action_conf *conf,
15105                       const struct rte_flow_action *action,
15106                       struct rte_flow_error *err)
15107 {
15108         struct mlx5_priv *priv = dev->data->dev_private;
15109         uint32_t age_idx = 0;
15110         uint32_t idx = 0;
15111         uint32_t ret = 0;
15112
15113         switch (action->type) {
15114         case RTE_FLOW_ACTION_TYPE_RSS:
15115                 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err);
15116                 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS <<
15117                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
15118                 break;
15119         case RTE_FLOW_ACTION_TYPE_AGE:
15120                 age_idx = flow_dv_aso_age_alloc(dev, err);
15121                 if (!age_idx) {
15122                         ret = -rte_errno;
15123                         break;
15124                 }
15125                 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
15126                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
15127                 flow_dv_aso_age_params_init(dev, age_idx,
15128                                         ((const struct rte_flow_action_age *)
15129                                                 action->conf)->context ?
15130                                         ((const struct rte_flow_action_age *)
15131                                                 action->conf)->context :
15132                                         (void *)(uintptr_t)idx,
15133                                         ((const struct rte_flow_action_age *)
15134                                                 action->conf)->timeout);
15135                 ret = age_idx;
15136                 break;
15137         case RTE_FLOW_ACTION_TYPE_COUNT:
15138                 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL);
15139                 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT <<
15140                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
15141                 break;
15142         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
15143                 ret = flow_dv_translate_create_conntrack(dev, action->conf,
15144                                                          err);
15145                 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret);
15146                 break;
15147         default:
15148                 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
15149                                    NULL, "action type not supported");
15150                 break;
15151         }
15152         return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL;
15153 }
15154
15155 /**
15156  * Destroy the indirect action.
15157  * Release action related resources on the NIC and the memory.
15158  * Lock free, (mutex should be acquired by caller).
15159  * Dispatcher for action type specific call.
15160  *
15161  * @param[in] dev
15162  *   Pointer to the Ethernet device structure.
15163  * @param[in] handle
15164  *   The indirect action object handle to be removed.
15165  * @param[out] error
15166  *   Perform verbose error reporting if not NULL. Initialized in case of
15167  *   error only.
15168  *
15169  * @return
15170  *   0 on success, otherwise negative errno value.
15171  */
15172 int
15173 flow_dv_action_destroy(struct rte_eth_dev *dev,
15174                        struct rte_flow_action_handle *handle,
15175                        struct rte_flow_error *error)
15176 {
15177         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15178         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15179         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15180         struct mlx5_flow_counter *cnt;
15181         uint32_t no_flow_refcnt = 1;
15182         int ret;
15183
15184         switch (type) {
15185         case MLX5_INDIRECT_ACTION_TYPE_RSS:
15186                 return __flow_dv_action_rss_release(dev, idx, error);
15187         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
15188                 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL);
15189                 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt,
15190                                                  &no_flow_refcnt, 1, false,
15191                                                  __ATOMIC_ACQUIRE,
15192                                                  __ATOMIC_RELAXED))
15193                         return rte_flow_error_set(error, EBUSY,
15194                                                   RTE_FLOW_ERROR_TYPE_ACTION,
15195                                                   NULL,
15196                                                   "Indirect count action has references");
15197                 flow_dv_counter_free(dev, idx);
15198                 return 0;
15199         case MLX5_INDIRECT_ACTION_TYPE_AGE:
15200                 ret = flow_dv_aso_age_release(dev, idx);
15201                 if (ret)
15202                         /*
15203                          * In this case, the last flow has a reference will
15204                          * actually release the age action.
15205                          */
15206                         DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was"
15207                                 " released with references %d.", idx, ret);
15208                 return 0;
15209         case MLX5_INDIRECT_ACTION_TYPE_CT:
15210                 ret = flow_dv_aso_ct_release(dev, idx, error);
15211                 if (ret < 0)
15212                         return ret;
15213                 if (ret > 0)
15214                         DRV_LOG(DEBUG, "Connection tracking object %u still "
15215                                 "has references %d.", idx, ret);
15216                 return 0;
15217         default:
15218                 return rte_flow_error_set(error, ENOTSUP,
15219                                           RTE_FLOW_ERROR_TYPE_ACTION,
15220                                           NULL,
15221                                           "action type not supported");
15222         }
15223 }
15224
15225 /**
15226  * Updates in place shared RSS action configuration.
15227  *
15228  * @param[in] dev
15229  *   Pointer to the Ethernet device structure.
15230  * @param[in] idx
15231  *   The shared RSS action object ID to be updated.
15232  * @param[in] action_conf
15233  *   RSS action specification used to modify *shared_rss*.
15234  * @param[out] error
15235  *   Perform verbose error reporting if not NULL. Initialized in case of
15236  *   error only.
15237  *
15238  * @return
15239  *   0 on success, otherwise negative errno value.
15240  * @note: currently only support update of RSS queues.
15241  */
15242 static int
15243 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx,
15244                             const struct rte_flow_action_rss *action_conf,
15245                             struct rte_flow_error *error)
15246 {
15247         struct mlx5_priv *priv = dev->data->dev_private;
15248         struct mlx5_shared_action_rss *shared_rss =
15249             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
15250         int ret = 0;
15251         void *queue = NULL;
15252         void *queue_i = NULL;
15253         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
15254         bool dev_started = !!dev->data->dev_started;
15255
15256         if (!shared_rss)
15257                 return rte_flow_error_set(error, EINVAL,
15258                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15259                                           "invalid shared action to update");
15260         if (priv->obj_ops.ind_table_modify == NULL)
15261                 return rte_flow_error_set(error, ENOTSUP,
15262                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15263                                           "cannot modify indirection table");
15264         queue = mlx5_malloc(MLX5_MEM_ZERO,
15265                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
15266                             0, SOCKET_ID_ANY);
15267         if (!queue)
15268                 return rte_flow_error_set(error, ENOMEM,
15269                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15270                                           NULL,
15271                                           "cannot allocate resource memory");
15272         memcpy(queue, action_conf->queue, queue_size);
15273         MLX5_ASSERT(shared_rss->ind_tbl);
15274         rte_spinlock_lock(&shared_rss->action_rss_sl);
15275         queue_i = shared_rss->ind_tbl->queues;
15276         ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl,
15277                                         queue, action_conf->queue_num,
15278                                         true /* standalone */,
15279                                         dev_started /* ref_new_qs */,
15280                                         dev_started /* deref_old_qs */);
15281         if (ret) {
15282                 ret = rte_flow_error_set(error, rte_errno,
15283                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15284                                           "cannot update indirection table");
15285         } else {
15286                 /* Restore the queue to indirect table internal queue. */
15287                 memcpy(queue_i, queue, queue_size);
15288                 shared_rss->ind_tbl->queues = queue_i;
15289                 shared_rss->origin.queue_num = action_conf->queue_num;
15290         }
15291         mlx5_free(queue);
15292         rte_spinlock_unlock(&shared_rss->action_rss_sl);
15293         return ret;
15294 }
15295
15296 /*
15297  * Updates in place conntrack context or direction.
15298  * Context update should be synchronized.
15299  *
15300  * @param[in] dev
15301  *   Pointer to the Ethernet device structure.
15302  * @param[in] idx
15303  *   The conntrack object ID to be updated.
15304  * @param[in] update
15305  *   Pointer to the structure of information to update.
15306  * @param[out] error
15307  *   Perform verbose error reporting if not NULL. Initialized in case of
15308  *   error only.
15309  *
15310  * @return
15311  *   0 on success, otherwise negative errno value.
15312  */
15313 static int
15314 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx,
15315                            const struct rte_flow_modify_conntrack *update,
15316                            struct rte_flow_error *error)
15317 {
15318         struct mlx5_priv *priv = dev->data->dev_private;
15319         struct mlx5_aso_ct_action *ct;
15320         const struct rte_flow_action_conntrack *new_prf;
15321         int ret = 0;
15322         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15323         uint32_t dev_idx;
15324
15325         if (PORT_ID(priv) != owner)
15326                 return rte_flow_error_set(error, EACCES,
15327                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15328                                           NULL,
15329                                           "CT object owned by another port");
15330         dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15331         ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15332         if (!ct->refcnt)
15333                 return rte_flow_error_set(error, ENOMEM,
15334                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15335                                           NULL,
15336                                           "CT object is inactive");
15337         new_prf = &update->new_ct;
15338         if (update->direction)
15339                 ct->is_original = !!new_prf->is_original_dir;
15340         if (update->state) {
15341                 /* Only validate the profile when it needs to be updated. */
15342                 ret = mlx5_validate_action_ct(dev, new_prf, error);
15343                 if (ret)
15344                         return ret;
15345                 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf);
15346                 if (ret)
15347                         return rte_flow_error_set(error, EIO,
15348                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15349                                         NULL,
15350                                         "Failed to send CT context update WQE");
15351                 /* Block until ready or a failure. */
15352                 ret = mlx5_aso_ct_available(priv->sh, ct);
15353                 if (ret)
15354                         rte_flow_error_set(error, rte_errno,
15355                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15356                                            NULL,
15357                                            "Timeout to get the CT update");
15358         }
15359         return ret;
15360 }
15361
15362 /**
15363  * Updates in place shared action configuration, lock free,
15364  * (mutex should be acquired by caller).
15365  *
15366  * @param[in] dev
15367  *   Pointer to the Ethernet device structure.
15368  * @param[in] handle
15369  *   The indirect action object handle to be updated.
15370  * @param[in] update
15371  *   Action specification used to modify the action pointed by *handle*.
15372  *   *update* could be of same type with the action pointed by the *handle*
15373  *   handle argument, or some other structures like a wrapper, depending on
15374  *   the indirect action type.
15375  * @param[out] error
15376  *   Perform verbose error reporting if not NULL. Initialized in case of
15377  *   error only.
15378  *
15379  * @return
15380  *   0 on success, otherwise negative errno value.
15381  */
15382 int
15383 flow_dv_action_update(struct rte_eth_dev *dev,
15384                         struct rte_flow_action_handle *handle,
15385                         const void *update,
15386                         struct rte_flow_error *err)
15387 {
15388         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15389         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15390         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15391         const void *action_conf;
15392
15393         switch (type) {
15394         case MLX5_INDIRECT_ACTION_TYPE_RSS:
15395                 action_conf = ((const struct rte_flow_action *)update)->conf;
15396                 return __flow_dv_action_rss_update(dev, idx, action_conf, err);
15397         case MLX5_INDIRECT_ACTION_TYPE_CT:
15398                 return __flow_dv_action_ct_update(dev, idx, update, err);
15399         default:
15400                 return rte_flow_error_set(err, ENOTSUP,
15401                                           RTE_FLOW_ERROR_TYPE_ACTION,
15402                                           NULL,
15403                                           "action type update not supported");
15404         }
15405 }
15406
15407 /**
15408  * Destroy the meter sub policy table rules.
15409  * Lock free, (mutex should be acquired by caller).
15410  *
15411  * @param[in] dev
15412  *   Pointer to Ethernet device.
15413  * @param[in] sub_policy
15414  *   Pointer to meter sub policy table.
15415  */
15416 static void
15417 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
15418                              struct mlx5_flow_meter_sub_policy *sub_policy)
15419 {
15420         struct mlx5_priv *priv = dev->data->dev_private;
15421         struct mlx5_flow_tbl_data_entry *tbl;
15422         struct mlx5_flow_meter_policy *policy = sub_policy->main_policy;
15423         struct mlx5_flow_meter_info *next_fm;
15424         struct mlx5_sub_policy_color_rule *color_rule;
15425         void *tmp;
15426         uint32_t i;
15427
15428         for (i = 0; i < RTE_COLORS; i++) {
15429                 next_fm = NULL;
15430                 if (i == RTE_COLOR_GREEN && policy &&
15431                     policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
15432                         next_fm = mlx5_flow_meter_find(priv,
15433                                         policy->act_cnt[i].next_mtr_id, NULL);
15434                 RTE_TAILQ_FOREACH_SAFE(color_rule, &sub_policy->color_rules[i],
15435                                    next_port, tmp) {
15436                         claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
15437                         tbl = container_of(color_rule->matcher->tbl,
15438                                            typeof(*tbl), tbl);
15439                         mlx5_list_unregister(tbl->matchers,
15440                                              &color_rule->matcher->entry);
15441                         TAILQ_REMOVE(&sub_policy->color_rules[i],
15442                                      color_rule, next_port);
15443                         mlx5_free(color_rule);
15444                         if (next_fm)
15445                                 mlx5_flow_meter_detach(priv, next_fm);
15446                 }
15447         }
15448         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15449                 if (sub_policy->rix_hrxq[i]) {
15450                         if (policy && !policy->is_hierarchy)
15451                                 mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]);
15452                         sub_policy->rix_hrxq[i] = 0;
15453                 }
15454                 if (sub_policy->jump_tbl[i]) {
15455                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15456                                                      sub_policy->jump_tbl[i]);
15457                         sub_policy->jump_tbl[i] = NULL;
15458                 }
15459         }
15460         if (sub_policy->tbl_rsc) {
15461                 flow_dv_tbl_resource_release(MLX5_SH(dev),
15462                                              sub_policy->tbl_rsc);
15463                 sub_policy->tbl_rsc = NULL;
15464         }
15465 }
15466
15467 /**
15468  * Destroy policy rules, lock free,
15469  * (mutex should be acquired by caller).
15470  * Dispatcher for action type specific call.
15471  *
15472  * @param[in] dev
15473  *   Pointer to the Ethernet device structure.
15474  * @param[in] mtr_policy
15475  *   Meter policy struct.
15476  */
15477 static void
15478 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev,
15479                              struct mlx5_flow_meter_policy *mtr_policy)
15480 {
15481         uint32_t i, j;
15482         struct mlx5_flow_meter_sub_policy *sub_policy;
15483         uint16_t sub_policy_num;
15484
15485         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15486                 sub_policy_num = (mtr_policy->sub_policy_num >>
15487                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15488                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15489                 for (j = 0; j < sub_policy_num; j++) {
15490                         sub_policy = mtr_policy->sub_policys[i][j];
15491                         if (sub_policy)
15492                                 __flow_dv_destroy_sub_policy_rules(dev,
15493                                                                    sub_policy);
15494                 }
15495         }
15496 }
15497
15498 /**
15499  * Destroy policy action, lock free,
15500  * (mutex should be acquired by caller).
15501  * Dispatcher for action type specific call.
15502  *
15503  * @param[in] dev
15504  *   Pointer to the Ethernet device structure.
15505  * @param[in] mtr_policy
15506  *   Meter policy struct.
15507  */
15508 static void
15509 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
15510                       struct mlx5_flow_meter_policy *mtr_policy)
15511 {
15512         struct rte_flow_action *rss_action;
15513         struct mlx5_flow_handle dev_handle;
15514         uint32_t i, j;
15515
15516         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15517                 if (mtr_policy->act_cnt[i].rix_mark) {
15518                         flow_dv_tag_release(dev,
15519                                 mtr_policy->act_cnt[i].rix_mark);
15520                         mtr_policy->act_cnt[i].rix_mark = 0;
15521                 }
15522                 if (mtr_policy->act_cnt[i].modify_hdr) {
15523                         dev_handle.dvh.modify_hdr =
15524                                 mtr_policy->act_cnt[i].modify_hdr;
15525                         flow_dv_modify_hdr_resource_release(dev, &dev_handle);
15526                 }
15527                 switch (mtr_policy->act_cnt[i].fate_action) {
15528                 case MLX5_FLOW_FATE_SHARED_RSS:
15529                         rss_action = mtr_policy->act_cnt[i].rss;
15530                         mlx5_free(rss_action);
15531                         break;
15532                 case MLX5_FLOW_FATE_PORT_ID:
15533                         if (mtr_policy->act_cnt[i].rix_port_id_action) {
15534                                 flow_dv_port_id_action_resource_release(dev,
15535                                 mtr_policy->act_cnt[i].rix_port_id_action);
15536                                 mtr_policy->act_cnt[i].rix_port_id_action = 0;
15537                         }
15538                         break;
15539                 case MLX5_FLOW_FATE_DROP:
15540                 case MLX5_FLOW_FATE_JUMP:
15541                         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15542                                 mtr_policy->act_cnt[i].dr_jump_action[j] =
15543                                                 NULL;
15544                         break;
15545                 default:
15546                         /*Queue action do nothing*/
15547                         break;
15548                 }
15549         }
15550         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15551                 mtr_policy->dr_drop_action[j] = NULL;
15552 }
15553
15554 /**
15555  * Create policy action per domain, lock free,
15556  * (mutex should be acquired by caller).
15557  * Dispatcher for action type specific call.
15558  *
15559  * @param[in] dev
15560  *   Pointer to the Ethernet device structure.
15561  * @param[in] mtr_policy
15562  *   Meter policy struct.
15563  * @param[in] action
15564  *   Action specification used to create meter actions.
15565  * @param[out] error
15566  *   Perform verbose error reporting if not NULL. Initialized in case of
15567  *   error only.
15568  *
15569  * @return
15570  *   0 on success, otherwise negative errno value.
15571  */
15572 static int
15573 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
15574                         struct mlx5_flow_meter_policy *mtr_policy,
15575                         const struct rte_flow_action *actions[RTE_COLORS],
15576                         enum mlx5_meter_domain domain,
15577                         struct rte_mtr_error *error)
15578 {
15579         struct mlx5_priv *priv = dev->data->dev_private;
15580         struct rte_flow_error flow_err;
15581         const struct rte_flow_action *act;
15582         uint64_t action_flags;
15583         struct mlx5_flow_handle dh;
15584         struct mlx5_flow dev_flow;
15585         struct mlx5_flow_dv_port_id_action_resource port_id_action;
15586         int i, ret;
15587         uint8_t egress, transfer;
15588         struct mlx5_meter_policy_action_container *act_cnt = NULL;
15589         union {
15590                 struct mlx5_flow_dv_modify_hdr_resource res;
15591                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
15592                             sizeof(struct mlx5_modification_cmd) *
15593                             (MLX5_MAX_MODIFY_NUM + 1)];
15594         } mhdr_dummy;
15595         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
15596
15597         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15598         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15599         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
15600         memset(&dev_flow, 0, sizeof(struct mlx5_flow));
15601         memset(&port_id_action, 0,
15602                sizeof(struct mlx5_flow_dv_port_id_action_resource));
15603         memset(mhdr_res, 0, sizeof(*mhdr_res));
15604         mhdr_res->ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
15605                                        (egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
15606                                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
15607         dev_flow.handle = &dh;
15608         dev_flow.dv.port_id_action = &port_id_action;
15609         dev_flow.external = true;
15610         for (i = 0; i < RTE_COLORS; i++) {
15611                 if (i < MLX5_MTR_RTE_COLORS)
15612                         act_cnt = &mtr_policy->act_cnt[i];
15613                 /* Skip the color policy actions creation. */
15614                 if ((i == RTE_COLOR_YELLOW && mtr_policy->skip_y) ||
15615                     (i == RTE_COLOR_GREEN && mtr_policy->skip_g))
15616                         continue;
15617                 action_flags = 0;
15618                 for (act = actions[i];
15619                      act && act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
15620                         switch (act->type) {
15621                         case RTE_FLOW_ACTION_TYPE_MARK:
15622                         {
15623                                 uint32_t tag_be = mlx5_flow_mark_set
15624                                         (((const struct rte_flow_action_mark *)
15625                                         (act->conf))->id);
15626
15627                                 if (i >= MLX5_MTR_RTE_COLORS)
15628                                         return -rte_mtr_error_set(error,
15629                                           ENOTSUP,
15630                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15631                                           NULL,
15632                                           "cannot create policy "
15633                                           "mark action for this color");
15634                                 if (flow_dv_tag_resource_register(dev, tag_be,
15635                                                   &dev_flow, &flow_err))
15636                                         return -rte_mtr_error_set(error,
15637                                         ENOTSUP,
15638                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15639                                         NULL,
15640                                         "cannot setup policy mark action");
15641                                 MLX5_ASSERT(dev_flow.dv.tag_resource);
15642                                 act_cnt->rix_mark =
15643                                         dev_flow.handle->dvh.rix_tag;
15644                                 action_flags |= MLX5_FLOW_ACTION_MARK;
15645                                 mtr_policy->mark = 1;
15646                                 break;
15647                         }
15648                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
15649                                 if (i >= MLX5_MTR_RTE_COLORS)
15650                                         return -rte_mtr_error_set(error,
15651                                           ENOTSUP,
15652                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15653                                           NULL,
15654                                           "cannot create policy "
15655                                           "set tag action for this color");
15656                                 if (flow_dv_convert_action_set_tag
15657                                 (dev, mhdr_res,
15658                                 (const struct rte_flow_action_set_tag *)
15659                                 act->conf,  &flow_err))
15660                                         return -rte_mtr_error_set(error,
15661                                         ENOTSUP,
15662                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15663                                         NULL, "cannot convert policy "
15664                                         "set tag action");
15665                                 if (!mhdr_res->actions_num)
15666                                         return -rte_mtr_error_set(error,
15667                                         ENOTSUP,
15668                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15669                                         NULL, "cannot find policy "
15670                                         "set tag action");
15671                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
15672                                 break;
15673                         case RTE_FLOW_ACTION_TYPE_DROP:
15674                         {
15675                                 struct mlx5_flow_mtr_mng *mtrmng =
15676                                                 priv->sh->mtrmng;
15677                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15678
15679                                 /*
15680                                  * Create the drop table with
15681                                  * METER DROP level.
15682                                  */
15683                                 if (!mtrmng->drop_tbl[domain]) {
15684                                         mtrmng->drop_tbl[domain] =
15685                                         flow_dv_tbl_resource_get(dev,
15686                                         MLX5_FLOW_TABLE_LEVEL_METER,
15687                                         egress, transfer, false, NULL, 0,
15688                                         0, MLX5_MTR_TABLE_ID_DROP, &flow_err);
15689                                         if (!mtrmng->drop_tbl[domain])
15690                                                 return -rte_mtr_error_set
15691                                         (error, ENOTSUP,
15692                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15693                                         NULL,
15694                                         "Failed to create meter drop table");
15695                                 }
15696                                 tbl_data = container_of
15697                                 (mtrmng->drop_tbl[domain],
15698                                 struct mlx5_flow_tbl_data_entry, tbl);
15699                                 if (i < MLX5_MTR_RTE_COLORS) {
15700                                         act_cnt->dr_jump_action[domain] =
15701                                                 tbl_data->jump.action;
15702                                         act_cnt->fate_action =
15703                                                 MLX5_FLOW_FATE_DROP;
15704                                 }
15705                                 if (i == RTE_COLOR_RED)
15706                                         mtr_policy->dr_drop_action[domain] =
15707                                                 tbl_data->jump.action;
15708                                 action_flags |= MLX5_FLOW_ACTION_DROP;
15709                                 break;
15710                         }
15711                         case RTE_FLOW_ACTION_TYPE_QUEUE:
15712                         {
15713                                 if (i >= MLX5_MTR_RTE_COLORS)
15714                                         return -rte_mtr_error_set(error,
15715                                         ENOTSUP,
15716                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15717                                         NULL, "cannot create policy "
15718                                         "fate queue for this color");
15719                                 act_cnt->queue =
15720                                 ((const struct rte_flow_action_queue *)
15721                                         (act->conf))->index;
15722                                 act_cnt->fate_action =
15723                                         MLX5_FLOW_FATE_QUEUE;
15724                                 dev_flow.handle->fate_action =
15725                                         MLX5_FLOW_FATE_QUEUE;
15726                                 mtr_policy->is_queue = 1;
15727                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
15728                                 break;
15729                         }
15730                         case RTE_FLOW_ACTION_TYPE_RSS:
15731                         {
15732                                 int rss_size;
15733
15734                                 if (i >= MLX5_MTR_RTE_COLORS)
15735                                         return -rte_mtr_error_set(error,
15736                                           ENOTSUP,
15737                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15738                                           NULL,
15739                                           "cannot create policy "
15740                                           "rss action for this color");
15741                                 /*
15742                                  * Save RSS conf into policy struct
15743                                  * for translate stage.
15744                                  */
15745                                 rss_size = (int)rte_flow_conv
15746                                         (RTE_FLOW_CONV_OP_ACTION,
15747                                         NULL, 0, act, &flow_err);
15748                                 if (rss_size <= 0)
15749                                         return -rte_mtr_error_set(error,
15750                                           ENOTSUP,
15751                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15752                                           NULL, "Get the wrong "
15753                                           "rss action struct size");
15754                                 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO,
15755                                                 rss_size, 0, SOCKET_ID_ANY);
15756                                 if (!act_cnt->rss)
15757                                         return -rte_mtr_error_set(error,
15758                                           ENOTSUP,
15759                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15760                                           NULL,
15761                                           "Fail to malloc rss action memory");
15762                                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION,
15763                                         act_cnt->rss, rss_size,
15764                                         act, &flow_err);
15765                                 if (ret < 0)
15766                                         return -rte_mtr_error_set(error,
15767                                           ENOTSUP,
15768                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15769                                           NULL, "Fail to save "
15770                                           "rss action into policy struct");
15771                                 act_cnt->fate_action =
15772                                         MLX5_FLOW_FATE_SHARED_RSS;
15773                                 action_flags |= MLX5_FLOW_ACTION_RSS;
15774                                 break;
15775                         }
15776                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
15777                         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
15778                         {
15779                                 struct mlx5_flow_dv_port_id_action_resource
15780                                         port_id_resource;
15781                                 uint32_t port_id = 0;
15782
15783                                 if (i >= MLX5_MTR_RTE_COLORS)
15784                                         return -rte_mtr_error_set(error,
15785                                         ENOTSUP,
15786                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15787                                         NULL, "cannot create policy "
15788                                         "port action for this color");
15789                                 memset(&port_id_resource, 0,
15790                                         sizeof(port_id_resource));
15791                                 if (flow_dv_translate_action_port_id(dev, act,
15792                                                 &port_id, &flow_err))
15793                                         return -rte_mtr_error_set(error,
15794                                         ENOTSUP,
15795                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15796                                         NULL, "cannot translate "
15797                                         "policy port action");
15798                                 port_id_resource.port_id = port_id;
15799                                 if (flow_dv_port_id_action_resource_register
15800                                         (dev, &port_id_resource,
15801                                         &dev_flow, &flow_err))
15802                                         return -rte_mtr_error_set(error,
15803                                         ENOTSUP,
15804                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15805                                         NULL, "cannot setup "
15806                                         "policy port action");
15807                                 act_cnt->rix_port_id_action =
15808                                         dev_flow.handle->rix_port_id_action;
15809                                 act_cnt->fate_action =
15810                                         MLX5_FLOW_FATE_PORT_ID;
15811                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
15812                                 break;
15813                         }
15814                         case RTE_FLOW_ACTION_TYPE_JUMP:
15815                         {
15816                                 uint32_t jump_group = 0;
15817                                 uint32_t table = 0;
15818                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15819                                 struct flow_grp_info grp_info = {
15820                                         .external = !!dev_flow.external,
15821                                         .transfer = !!transfer,
15822                                         .fdb_def_rule = !!priv->fdb_def_rule,
15823                                         .std_tbl_fix = 0,
15824                                         .skip_scale = dev_flow.skip_scale &
15825                                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
15826                                 };
15827                                 struct mlx5_flow_meter_sub_policy *sub_policy =
15828                                         mtr_policy->sub_policys[domain][0];
15829
15830                                 if (i >= MLX5_MTR_RTE_COLORS)
15831                                         return -rte_mtr_error_set(error,
15832                                           ENOTSUP,
15833                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15834                                           NULL,
15835                                           "cannot create policy "
15836                                           "jump action for this color");
15837                                 jump_group =
15838                                 ((const struct rte_flow_action_jump *)
15839                                                         act->conf)->group;
15840                                 if (mlx5_flow_group_to_table(dev, NULL,
15841                                                        jump_group,
15842                                                        &table,
15843                                                        &grp_info, &flow_err))
15844                                         return -rte_mtr_error_set(error,
15845                                         ENOTSUP,
15846                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15847                                         NULL, "cannot setup "
15848                                         "policy jump action");
15849                                 sub_policy->jump_tbl[i] =
15850                                 flow_dv_tbl_resource_get(dev,
15851                                         table, egress,
15852                                         transfer,
15853                                         !!dev_flow.external,
15854                                         NULL, jump_group, 0,
15855                                         0, &flow_err);
15856                                 if
15857                                 (!sub_policy->jump_tbl[i])
15858                                         return  -rte_mtr_error_set(error,
15859                                         ENOTSUP,
15860                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15861                                         NULL, "cannot create jump action.");
15862                                 tbl_data = container_of
15863                                 (sub_policy->jump_tbl[i],
15864                                 struct mlx5_flow_tbl_data_entry, tbl);
15865                                 act_cnt->dr_jump_action[domain] =
15866                                         tbl_data->jump.action;
15867                                 act_cnt->fate_action =
15868                                         MLX5_FLOW_FATE_JUMP;
15869                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
15870                                 break;
15871                         }
15872                         /*
15873                          * No need to check meter hierarchy for Y or R colors
15874                          * here since it is done in the validation stage.
15875                          */
15876                         case RTE_FLOW_ACTION_TYPE_METER:
15877                         {
15878                                 const struct rte_flow_action_meter *mtr;
15879                                 struct mlx5_flow_meter_info *next_fm;
15880                                 struct mlx5_flow_meter_policy *next_policy;
15881                                 struct rte_flow_action tag_action;
15882                                 struct mlx5_rte_flow_action_set_tag set_tag;
15883                                 uint32_t next_mtr_idx = 0;
15884
15885                                 mtr = act->conf;
15886                                 next_fm = mlx5_flow_meter_find(priv,
15887                                                         mtr->mtr_id,
15888                                                         &next_mtr_idx);
15889                                 if (!next_fm)
15890                                         return -rte_mtr_error_set(error, EINVAL,
15891                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15892                                                 "Fail to find next meter.");
15893                                 if (next_fm->def_policy)
15894                                         return -rte_mtr_error_set(error, EINVAL,
15895                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15896                                 "Hierarchy only supports termination meter.");
15897                                 next_policy = mlx5_flow_meter_policy_find(dev,
15898                                                 next_fm->policy_id, NULL);
15899                                 MLX5_ASSERT(next_policy);
15900                                 if (next_fm->drop_cnt) {
15901                                         set_tag.id =
15902                                                 (enum modify_reg)
15903                                                 mlx5_flow_get_reg_id(dev,
15904                                                 MLX5_MTR_ID,
15905                                                 0,
15906                                                 (struct rte_flow_error *)error);
15907                                         set_tag.offset = (priv->mtr_reg_share ?
15908                                                 MLX5_MTR_COLOR_BITS : 0);
15909                                         set_tag.length = (priv->mtr_reg_share ?
15910                                                MLX5_MTR_IDLE_BITS_IN_COLOR_REG :
15911                                                MLX5_REG_BITS);
15912                                         set_tag.data = next_mtr_idx;
15913                                         tag_action.type =
15914                                                 (enum rte_flow_action_type)
15915                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
15916                                         tag_action.conf = &set_tag;
15917                                         if (flow_dv_convert_action_set_reg
15918                                                 (mhdr_res, &tag_action,
15919                                                 (struct rte_flow_error *)error))
15920                                                 return -rte_errno;
15921                                         action_flags |=
15922                                                 MLX5_FLOW_ACTION_SET_TAG;
15923                                 }
15924                                 act_cnt->fate_action = MLX5_FLOW_FATE_MTR;
15925                                 act_cnt->next_mtr_id = next_fm->meter_id;
15926                                 act_cnt->next_sub_policy = NULL;
15927                                 mtr_policy->is_hierarchy = 1;
15928                                 mtr_policy->dev = next_policy->dev;
15929                                 if (next_policy->mark)
15930                                         mtr_policy->mark = 1;
15931                                 action_flags |=
15932                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
15933                                 break;
15934                         }
15935                         default:
15936                                 return -rte_mtr_error_set(error, ENOTSUP,
15937                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15938                                           NULL, "action type not supported");
15939                         }
15940                         if (action_flags & MLX5_FLOW_ACTION_SET_TAG) {
15941                                 /* create modify action if needed. */
15942                                 dev_flow.dv.group = 1;
15943                                 if (flow_dv_modify_hdr_resource_register
15944                                         (dev, mhdr_res, &dev_flow, &flow_err))
15945                                         return -rte_mtr_error_set(error,
15946                                                 ENOTSUP,
15947                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
15948                                                 NULL, "cannot register policy "
15949                                                 "set tag action");
15950                                 act_cnt->modify_hdr =
15951                                         dev_flow.handle->dvh.modify_hdr;
15952                         }
15953                 }
15954         }
15955         return 0;
15956 }
15957
15958 /**
15959  * Create policy action per domain, lock free,
15960  * (mutex should be acquired by caller).
15961  * Dispatcher for action type specific call.
15962  *
15963  * @param[in] dev
15964  *   Pointer to the Ethernet device structure.
15965  * @param[in] mtr_policy
15966  *   Meter policy struct.
15967  * @param[in] action
15968  *   Action specification used to create meter actions.
15969  * @param[out] error
15970  *   Perform verbose error reporting if not NULL. Initialized in case of
15971  *   error only.
15972  *
15973  * @return
15974  *   0 on success, otherwise negative errno value.
15975  */
15976 static int
15977 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev,
15978                       struct mlx5_flow_meter_policy *mtr_policy,
15979                       const struct rte_flow_action *actions[RTE_COLORS],
15980                       struct rte_mtr_error *error)
15981 {
15982         int ret, i;
15983         uint16_t sub_policy_num;
15984
15985         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15986                 sub_policy_num = (mtr_policy->sub_policy_num >>
15987                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15988                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15989                 if (sub_policy_num) {
15990                         ret = __flow_dv_create_domain_policy_acts(dev,
15991                                 mtr_policy, actions,
15992                                 (enum mlx5_meter_domain)i, error);
15993                         /* Cleaning resource is done in the caller level. */
15994                         if (ret)
15995                                 return ret;
15996                 }
15997         }
15998         return 0;
15999 }
16000
16001 /**
16002  * Query a DV flow rule for its statistics via DevX.
16003  *
16004  * @param[in] dev
16005  *   Pointer to Ethernet device.
16006  * @param[in] cnt_idx
16007  *   Index to the flow counter.
16008  * @param[out] data
16009  *   Data retrieved by the query.
16010  * @param[out] error
16011  *   Perform verbose error reporting if not NULL.
16012  *
16013  * @return
16014  *   0 on success, a negative errno value otherwise and rte_errno is set.
16015  */
16016 static int
16017 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data,
16018                     struct rte_flow_error *error)
16019 {
16020         struct mlx5_priv *priv = dev->data->dev_private;
16021         struct rte_flow_query_count *qc = data;
16022
16023         if (!priv->sh->cdev->config.devx)
16024                 return rte_flow_error_set(error, ENOTSUP,
16025                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16026                                           NULL,
16027                                           "counters are not supported");
16028         if (cnt_idx) {
16029                 uint64_t pkts, bytes;
16030                 struct mlx5_flow_counter *cnt;
16031                 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes);
16032
16033                 if (err)
16034                         return rte_flow_error_set(error, -err,
16035                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16036                                         NULL, "cannot read counters");
16037                 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL);
16038                 qc->hits_set = 1;
16039                 qc->bytes_set = 1;
16040                 qc->hits = pkts - cnt->hits;
16041                 qc->bytes = bytes - cnt->bytes;
16042                 if (qc->reset) {
16043                         cnt->hits = pkts;
16044                         cnt->bytes = bytes;
16045                 }
16046                 return 0;
16047         }
16048         return rte_flow_error_set(error, EINVAL,
16049                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16050                                   NULL,
16051                                   "counters are not available");
16052 }
16053
16054 int
16055 flow_dv_action_query(struct rte_eth_dev *dev,
16056                      const struct rte_flow_action_handle *handle, void *data,
16057                      struct rte_flow_error *error)
16058 {
16059         struct mlx5_age_param *age_param;
16060         struct rte_flow_query_age *resp;
16061         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
16062         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
16063         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
16064         struct mlx5_priv *priv = dev->data->dev_private;
16065         struct mlx5_aso_ct_action *ct;
16066         uint16_t owner;
16067         uint32_t dev_idx;
16068
16069         switch (type) {
16070         case MLX5_INDIRECT_ACTION_TYPE_AGE:
16071                 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params;
16072                 resp = data;
16073                 resp->aged = __atomic_load_n(&age_param->state,
16074                                               __ATOMIC_RELAXED) == AGE_TMOUT ?
16075                                                                           1 : 0;
16076                 resp->sec_since_last_hit_valid = !resp->aged;
16077                 if (resp->sec_since_last_hit_valid)
16078                         resp->sec_since_last_hit = __atomic_load_n
16079                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
16080                 return 0;
16081         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
16082                 return flow_dv_query_count(dev, idx, data, error);
16083         case MLX5_INDIRECT_ACTION_TYPE_CT:
16084                 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
16085                 if (owner != PORT_ID(priv))
16086                         return rte_flow_error_set(error, EACCES,
16087                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16088                                         NULL,
16089                                         "CT object owned by another port");
16090                 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
16091                 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
16092                 MLX5_ASSERT(ct);
16093                 if (!ct->refcnt)
16094                         return rte_flow_error_set(error, EFAULT,
16095                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16096                                         NULL,
16097                                         "CT object is inactive");
16098                 ((struct rte_flow_action_conntrack *)data)->peer_port =
16099                                                         ct->peer;
16100                 ((struct rte_flow_action_conntrack *)data)->is_original_dir =
16101                                                         ct->is_original;
16102                 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data))
16103                         return rte_flow_error_set(error, EIO,
16104                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16105                                         NULL,
16106                                         "Failed to query CT context");
16107                 return 0;
16108         default:
16109                 return rte_flow_error_set(error, ENOTSUP,
16110                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
16111                                           "action type query not supported");
16112         }
16113 }
16114
16115 /**
16116  * Query a flow rule AGE action for aging information.
16117  *
16118  * @param[in] dev
16119  *   Pointer to Ethernet device.
16120  * @param[in] flow
16121  *   Pointer to the sub flow.
16122  * @param[out] data
16123  *   data retrieved by the query.
16124  * @param[out] error
16125  *   Perform verbose error reporting if not NULL.
16126  *
16127  * @return
16128  *   0 on success, a negative errno value otherwise and rte_errno is set.
16129  */
16130 static int
16131 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
16132                   void *data, struct rte_flow_error *error)
16133 {
16134         struct rte_flow_query_age *resp = data;
16135         struct mlx5_age_param *age_param;
16136
16137         if (flow->age) {
16138                 struct mlx5_aso_age_action *act =
16139                                      flow_aso_age_get_by_idx(dev, flow->age);
16140
16141                 age_param = &act->age_params;
16142         } else if (flow->counter) {
16143                 age_param = flow_dv_counter_idx_get_age(dev, flow->counter);
16144
16145                 if (!age_param || !age_param->timeout)
16146                         return rte_flow_error_set
16147                                         (error, EINVAL,
16148                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16149                                          NULL, "cannot read age data");
16150         } else {
16151                 return rte_flow_error_set(error, EINVAL,
16152                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16153                                           NULL, "age data not available");
16154         }
16155         resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
16156                                      AGE_TMOUT ? 1 : 0;
16157         resp->sec_since_last_hit_valid = !resp->aged;
16158         if (resp->sec_since_last_hit_valid)
16159                 resp->sec_since_last_hit = __atomic_load_n
16160                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
16161         return 0;
16162 }
16163
16164 /**
16165  * Query a flow.
16166  *
16167  * @see rte_flow_query()
16168  * @see rte_flow_ops
16169  */
16170 static int
16171 flow_dv_query(struct rte_eth_dev *dev,
16172               struct rte_flow *flow __rte_unused,
16173               const struct rte_flow_action *actions __rte_unused,
16174               void *data __rte_unused,
16175               struct rte_flow_error *error __rte_unused)
16176 {
16177         int ret = -EINVAL;
16178
16179         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
16180                 switch (actions->type) {
16181                 case RTE_FLOW_ACTION_TYPE_VOID:
16182                         break;
16183                 case RTE_FLOW_ACTION_TYPE_COUNT:
16184                         ret = flow_dv_query_count(dev, flow->counter, data,
16185                                                   error);
16186                         break;
16187                 case RTE_FLOW_ACTION_TYPE_AGE:
16188                         ret = flow_dv_query_age(dev, flow, data, error);
16189                         break;
16190                 default:
16191                         return rte_flow_error_set(error, ENOTSUP,
16192                                                   RTE_FLOW_ERROR_TYPE_ACTION,
16193                                                   actions,
16194                                                   "action not supported");
16195                 }
16196         }
16197         return ret;
16198 }
16199
16200 /**
16201  * Destroy the meter table set.
16202  * Lock free, (mutex should be acquired by caller).
16203  *
16204  * @param[in] dev
16205  *   Pointer to Ethernet device.
16206  * @param[in] fm
16207  *   Meter information table.
16208  */
16209 static void
16210 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
16211                         struct mlx5_flow_meter_info *fm)
16212 {
16213         struct mlx5_priv *priv = dev->data->dev_private;
16214         int i;
16215
16216         if (!fm || !priv->sh->config.dv_flow_en)
16217                 return;
16218         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16219                 if (fm->drop_rule[i]) {
16220                         claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
16221                         fm->drop_rule[i] = NULL;
16222                 }
16223         }
16224 }
16225
16226 static void
16227 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
16228 {
16229         struct mlx5_priv *priv = dev->data->dev_private;
16230         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16231         struct mlx5_flow_tbl_data_entry *tbl;
16232         int i, j;
16233
16234         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16235                 if (mtrmng->def_rule[i]) {
16236                         claim_zero(mlx5_flow_os_destroy_flow
16237                                         (mtrmng->def_rule[i]));
16238                         mtrmng->def_rule[i] = NULL;
16239                 }
16240                 if (mtrmng->def_matcher[i]) {
16241                         tbl = container_of(mtrmng->def_matcher[i]->tbl,
16242                                 struct mlx5_flow_tbl_data_entry, tbl);
16243                         mlx5_list_unregister(tbl->matchers,
16244                                              &mtrmng->def_matcher[i]->entry);
16245                         mtrmng->def_matcher[i] = NULL;
16246                 }
16247                 for (j = 0; j < MLX5_REG_BITS; j++) {
16248                         if (mtrmng->drop_matcher[i][j]) {
16249                                 tbl =
16250                                 container_of(mtrmng->drop_matcher[i][j]->tbl,
16251                                              struct mlx5_flow_tbl_data_entry,
16252                                              tbl);
16253                                 mlx5_list_unregister(tbl->matchers,
16254                                             &mtrmng->drop_matcher[i][j]->entry);
16255                                 mtrmng->drop_matcher[i][j] = NULL;
16256                         }
16257                 }
16258                 if (mtrmng->drop_tbl[i]) {
16259                         flow_dv_tbl_resource_release(MLX5_SH(dev),
16260                                 mtrmng->drop_tbl[i]);
16261                         mtrmng->drop_tbl[i] = NULL;
16262                 }
16263         }
16264 }
16265
16266 /* Number of meter flow actions, count and jump or count and drop. */
16267 #define METER_ACTIONS 2
16268
16269 static void
16270 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
16271                                     enum mlx5_meter_domain domain)
16272 {
16273         struct mlx5_priv *priv = dev->data->dev_private;
16274         struct mlx5_flow_meter_def_policy *def_policy =
16275                         priv->sh->mtrmng->def_policy[domain];
16276
16277         __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
16278         mlx5_free(def_policy);
16279         priv->sh->mtrmng->def_policy[domain] = NULL;
16280 }
16281
16282 /**
16283  * Destroy the default policy table set.
16284  *
16285  * @param[in] dev
16286  *   Pointer to Ethernet device.
16287  */
16288 static void
16289 flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
16290 {
16291         struct mlx5_priv *priv = dev->data->dev_private;
16292         int i;
16293
16294         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
16295                 if (priv->sh->mtrmng->def_policy[i])
16296                         __flow_dv_destroy_domain_def_policy(dev,
16297                                         (enum mlx5_meter_domain)i);
16298         priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
16299 }
16300
16301 static int
16302 __flow_dv_create_policy_flow(struct rte_eth_dev *dev,
16303                         uint32_t color_reg_c_idx,
16304                         enum rte_color color, void *matcher_object,
16305                         int actions_n, void *actions,
16306                         bool match_src_port, const struct rte_flow_item *item,
16307                         void **rule, const struct rte_flow_attr *attr)
16308 {
16309         int ret;
16310         struct mlx5_flow_dv_match_params value = {
16311                 .size = sizeof(value.buf),
16312         };
16313         struct mlx5_flow_dv_match_params matcher = {
16314                 .size = sizeof(matcher.buf),
16315         };
16316         struct mlx5_priv *priv = dev->data->dev_private;
16317         uint8_t misc_mask;
16318
16319         if (match_src_port && priv->sh->esw_mode) {
16320                 if (flow_dv_translate_item_port_id(dev, matcher.buf,
16321                                                    value.buf, item, attr)) {
16322                         DRV_LOG(ERR, "Failed to create meter policy%d flow's"
16323                                 " value with port.", color);
16324                         return -1;
16325                 }
16326         }
16327         flow_dv_match_meta_reg(matcher.buf, value.buf,
16328                                (enum modify_reg)color_reg_c_idx,
16329                                rte_col_2_mlx5_col(color), UINT32_MAX);
16330         misc_mask = flow_dv_matcher_enable(value.buf);
16331         __flow_dv_adjust_buf_size(&value.size, misc_mask);
16332         ret = mlx5_flow_os_create_flow(matcher_object, (void *)&value,
16333                                        actions_n, actions, rule);
16334         if (ret) {
16335                 DRV_LOG(ERR, "Failed to create meter policy%d flow.", color);
16336                 return -1;
16337         }
16338         return 0;
16339 }
16340
16341 static int
16342 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
16343                         uint32_t color_reg_c_idx,
16344                         uint16_t priority,
16345                         struct mlx5_flow_meter_sub_policy *sub_policy,
16346                         const struct rte_flow_attr *attr,
16347                         bool match_src_port,
16348                         const struct rte_flow_item *item,
16349                         struct mlx5_flow_dv_matcher **policy_matcher,
16350                         struct rte_flow_error *error)
16351 {
16352         struct mlx5_list_entry *entry;
16353         struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
16354         struct mlx5_flow_dv_matcher matcher = {
16355                 .mask = {
16356                         .size = sizeof(matcher.mask.buf),
16357                 },
16358                 .tbl = tbl_rsc,
16359         };
16360         struct mlx5_flow_dv_match_params value = {
16361                 .size = sizeof(value.buf),
16362         };
16363         struct mlx5_flow_cb_ctx ctx = {
16364                 .error = error,
16365                 .data = &matcher,
16366         };
16367         struct mlx5_flow_tbl_data_entry *tbl_data;
16368         struct mlx5_priv *priv = dev->data->dev_private;
16369         const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
16370
16371         if (match_src_port && priv->sh->esw_mode) {
16372                 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
16373                                                    value.buf, item, attr)) {
16374                         DRV_LOG(ERR, "Failed to register meter policy%d matcher"
16375                                 " with port.", priority);
16376                         return -1;
16377                 }
16378         }
16379         tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
16380         if (priority < RTE_COLOR_RED)
16381                 flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16382                         (enum modify_reg)color_reg_c_idx, 0, color_mask);
16383         matcher.priority = priority;
16384         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
16385                                     matcher.mask.size);
16386         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16387         if (!entry) {
16388                 DRV_LOG(ERR, "Failed to register meter drop matcher.");
16389                 return -1;
16390         }
16391         *policy_matcher =
16392                 container_of(entry, struct mlx5_flow_dv_matcher, entry);
16393         return 0;
16394 }
16395
16396 /**
16397  * Create the policy rules per domain.
16398  *
16399  * @param[in] dev
16400  *   Pointer to Ethernet device.
16401  * @param[in] sub_policy
16402  *    Pointer to sub policy table..
16403  * @param[in] egress
16404  *   Direction of the table.
16405  * @param[in] transfer
16406  *   E-Switch or NIC flow.
16407  * @param[in] acts
16408  *   Pointer to policy action list per color.
16409  *
16410  * @return
16411  *   0 on success, -1 otherwise.
16412  */
16413 static int
16414 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
16415                 struct mlx5_flow_meter_sub_policy *sub_policy,
16416                 uint8_t egress, uint8_t transfer, bool match_src_port,
16417                 struct mlx5_meter_policy_acts acts[RTE_COLORS])
16418 {
16419         struct mlx5_priv *priv = dev->data->dev_private;
16420         struct rte_flow_error flow_err;
16421         uint32_t color_reg_c_idx;
16422         struct rte_flow_attr attr = {
16423                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16424                 .priority = 0,
16425                 .ingress = 0,
16426                 .egress = !!egress,
16427                 .transfer = !!transfer,
16428                 .reserved = 0,
16429         };
16430         int i;
16431         int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
16432         struct mlx5_sub_policy_color_rule *color_rule;
16433         bool svport_match;
16434         struct mlx5_sub_policy_color_rule *tmp_rules[RTE_COLORS] = {NULL};
16435
16436         if (ret < 0)
16437                 return -1;
16438         /* Create policy table with POLICY level. */
16439         if (!sub_policy->tbl_rsc)
16440                 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
16441                                 MLX5_FLOW_TABLE_LEVEL_POLICY,
16442                                 egress, transfer, false, NULL, 0, 0,
16443                                 sub_policy->idx, &flow_err);
16444         if (!sub_policy->tbl_rsc) {
16445                 DRV_LOG(ERR,
16446                         "Failed to create meter sub policy table.");
16447                 return -1;
16448         }
16449         /* Prepare matchers. */
16450         color_reg_c_idx = ret;
16451         for (i = 0; i < RTE_COLORS; i++) {
16452                 TAILQ_INIT(&sub_policy->color_rules[i]);
16453                 if (!acts[i].actions_n)
16454                         continue;
16455                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
16456                                 sizeof(struct mlx5_sub_policy_color_rule),
16457                                 0, SOCKET_ID_ANY);
16458                 if (!color_rule) {
16459                         DRV_LOG(ERR, "No memory to create color rule.");
16460                         goto err_exit;
16461                 }
16462                 tmp_rules[i] = color_rule;
16463                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
16464                                   color_rule, next_port);
16465                 color_rule->src_port = priv->representor_id;
16466                 /* No use. */
16467                 attr.priority = i;
16468                 /* Create matchers for colors. */
16469                 svport_match = (i != RTE_COLOR_RED) ? match_src_port : false;
16470                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
16471                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
16472                                 &attr, svport_match, NULL,
16473                                 &color_rule->matcher, &flow_err)) {
16474                         DRV_LOG(ERR, "Failed to create color%u matcher.", i);
16475                         goto err_exit;
16476                 }
16477                 /* Create flow, matching color. */
16478                 if (__flow_dv_create_policy_flow(dev,
16479                                 color_reg_c_idx, (enum rte_color)i,
16480                                 color_rule->matcher->matcher_object,
16481                                 acts[i].actions_n, acts[i].dv_actions,
16482                                 svport_match, NULL, &color_rule->rule,
16483                                 &attr)) {
16484                         DRV_LOG(ERR, "Failed to create color%u rule.", i);
16485                         goto err_exit;
16486                 }
16487         }
16488         return 0;
16489 err_exit:
16490         /* All the policy rules will be cleared. */
16491         do {
16492                 color_rule = tmp_rules[i];
16493                 if (color_rule) {
16494                         if (color_rule->rule)
16495                                 mlx5_flow_os_destroy_flow(color_rule->rule);
16496                         if (color_rule->matcher) {
16497                                 struct mlx5_flow_tbl_data_entry *tbl =
16498                                         container_of(color_rule->matcher->tbl,
16499                                                      typeof(*tbl), tbl);
16500                                 mlx5_list_unregister(tbl->matchers,
16501                                                 &color_rule->matcher->entry);
16502                         }
16503                         TAILQ_REMOVE(&sub_policy->color_rules[i],
16504                                      color_rule, next_port);
16505                         mlx5_free(color_rule);
16506                 }
16507         } while (i--);
16508         return -1;
16509 }
16510
16511 static int
16512 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
16513                         struct mlx5_flow_meter_policy *mtr_policy,
16514                         struct mlx5_flow_meter_sub_policy *sub_policy,
16515                         uint32_t domain)
16516 {
16517         struct mlx5_priv *priv = dev->data->dev_private;
16518         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16519         struct mlx5_flow_dv_tag_resource *tag;
16520         struct mlx5_flow_dv_port_id_action_resource *port_action;
16521         struct mlx5_hrxq *hrxq;
16522         struct mlx5_flow_meter_info *next_fm = NULL;
16523         struct mlx5_flow_meter_policy *next_policy;
16524         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16525         struct mlx5_flow_tbl_data_entry *tbl_data;
16526         struct rte_flow_error error;
16527         uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16528         uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16529         bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX);
16530         bool match_src_port = false;
16531         int i;
16532
16533         /* If RSS or Queue, no previous actions / rules is created. */
16534         for (i = 0; i < RTE_COLORS; i++) {
16535                 acts[i].actions_n = 0;
16536                 if (i == RTE_COLOR_RED) {
16537                         /* Only support drop on red. */
16538                         acts[i].dv_actions[0] =
16539                                 mtr_policy->dr_drop_action[domain];
16540                         acts[i].actions_n = 1;
16541                         continue;
16542                 }
16543                 if (i == RTE_COLOR_GREEN &&
16544                     mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
16545                         struct rte_flow_attr attr = {
16546                                 .transfer = transfer
16547                         };
16548
16549                         next_fm = mlx5_flow_meter_find(priv,
16550                                         mtr_policy->act_cnt[i].next_mtr_id,
16551                                         NULL);
16552                         if (!next_fm) {
16553                                 DRV_LOG(ERR,
16554                                         "Failed to get next hierarchy meter.");
16555                                 goto err_exit;
16556                         }
16557                         if (mlx5_flow_meter_attach(priv, next_fm,
16558                                                    &attr, &error)) {
16559                                 DRV_LOG(ERR, "%s", error.message);
16560                                 next_fm = NULL;
16561                                 goto err_exit;
16562                         }
16563                         /* Meter action must be the first for TX. */
16564                         if (mtr_first) {
16565                                 acts[i].dv_actions[acts[i].actions_n] =
16566                                         next_fm->meter_action;
16567                                 acts[i].actions_n++;
16568                         }
16569                 }
16570                 if (mtr_policy->act_cnt[i].rix_mark) {
16571                         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
16572                                         mtr_policy->act_cnt[i].rix_mark);
16573                         if (!tag) {
16574                                 DRV_LOG(ERR, "Failed to find "
16575                                 "mark action for policy.");
16576                                 goto err_exit;
16577                         }
16578                         acts[i].dv_actions[acts[i].actions_n] = tag->action;
16579                         acts[i].actions_n++;
16580                 }
16581                 if (mtr_policy->act_cnt[i].modify_hdr) {
16582                         acts[i].dv_actions[acts[i].actions_n] =
16583                                 mtr_policy->act_cnt[i].modify_hdr->action;
16584                         acts[i].actions_n++;
16585                 }
16586                 if (mtr_policy->act_cnt[i].fate_action) {
16587                         switch (mtr_policy->act_cnt[i].fate_action) {
16588                         case MLX5_FLOW_FATE_PORT_ID:
16589                                 port_action = mlx5_ipool_get
16590                                         (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
16591                                 mtr_policy->act_cnt[i].rix_port_id_action);
16592                                 if (!port_action) {
16593                                         DRV_LOG(ERR, "Failed to find "
16594                                                 "port action for policy.");
16595                                         goto err_exit;
16596                                 }
16597                                 acts[i].dv_actions[acts[i].actions_n] =
16598                                         port_action->action;
16599                                 acts[i].actions_n++;
16600                                 mtr_policy->dev = dev;
16601                                 match_src_port = true;
16602                                 break;
16603                         case MLX5_FLOW_FATE_DROP:
16604                         case MLX5_FLOW_FATE_JUMP:
16605                                 acts[i].dv_actions[acts[i].actions_n] =
16606                                 mtr_policy->act_cnt[i].dr_jump_action[domain];
16607                                 acts[i].actions_n++;
16608                                 break;
16609                         case MLX5_FLOW_FATE_SHARED_RSS:
16610                         case MLX5_FLOW_FATE_QUEUE:
16611                                 hrxq = mlx5_ipool_get
16612                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
16613                                          sub_policy->rix_hrxq[i]);
16614                                 if (!hrxq) {
16615                                         DRV_LOG(ERR, "Failed to find "
16616                                                 "queue action for policy.");
16617                                         goto err_exit;
16618                                 }
16619                                 acts[i].dv_actions[acts[i].actions_n] =
16620                                         hrxq->action;
16621                                 acts[i].actions_n++;
16622                                 break;
16623                         case MLX5_FLOW_FATE_MTR:
16624                                 if (!next_fm) {
16625                                         DRV_LOG(ERR,
16626                                                 "No next hierarchy meter.");
16627                                         goto err_exit;
16628                                 }
16629                                 if (!mtr_first) {
16630                                         acts[i].dv_actions[acts[i].actions_n] =
16631                                                         next_fm->meter_action;
16632                                         acts[i].actions_n++;
16633                                 }
16634                                 if (mtr_policy->act_cnt[i].next_sub_policy) {
16635                                         next_sub_policy =
16636                                         mtr_policy->act_cnt[i].next_sub_policy;
16637                                 } else {
16638                                         next_policy =
16639                                                 mlx5_flow_meter_policy_find(dev,
16640                                                 next_fm->policy_id, NULL);
16641                                         MLX5_ASSERT(next_policy);
16642                                         next_sub_policy =
16643                                         next_policy->sub_policys[domain][0];
16644                                 }
16645                                 tbl_data =
16646                                         container_of(next_sub_policy->tbl_rsc,
16647                                         struct mlx5_flow_tbl_data_entry, tbl);
16648                                 acts[i].dv_actions[acts[i].actions_n++] =
16649                                                         tbl_data->jump.action;
16650                                 if (mtr_policy->act_cnt[i].modify_hdr)
16651                                         match_src_port = !!transfer;
16652                                 break;
16653                         default:
16654                                 /*Queue action do nothing*/
16655                                 break;
16656                         }
16657                 }
16658         }
16659         if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
16660                                 egress, transfer, match_src_port, acts)) {
16661                 DRV_LOG(ERR,
16662                         "Failed to create policy rules per domain.");
16663                 goto err_exit;
16664         }
16665         return 0;
16666 err_exit:
16667         if (next_fm)
16668                 mlx5_flow_meter_detach(priv, next_fm);
16669         return -1;
16670 }
16671
16672 /**
16673  * Create the policy rules.
16674  *
16675  * @param[in] dev
16676  *   Pointer to Ethernet device.
16677  * @param[in,out] mtr_policy
16678  *   Pointer to meter policy table.
16679  *
16680  * @return
16681  *   0 on success, -1 otherwise.
16682  */
16683 static int
16684 flow_dv_create_policy_rules(struct rte_eth_dev *dev,
16685                              struct mlx5_flow_meter_policy *mtr_policy)
16686 {
16687         int i;
16688         uint16_t sub_policy_num;
16689
16690         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16691                 sub_policy_num = (mtr_policy->sub_policy_num >>
16692                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
16693                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16694                 if (!sub_policy_num)
16695                         continue;
16696                 /* Prepare actions list and create policy rules. */
16697                 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16698                         mtr_policy->sub_policys[i][0], i)) {
16699                         DRV_LOG(ERR, "Failed to create policy action "
16700                                 "list per domain.");
16701                         return -1;
16702                 }
16703         }
16704         return 0;
16705 }
16706
16707 static int
16708 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
16709 {
16710         struct mlx5_priv *priv = dev->data->dev_private;
16711         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16712         struct mlx5_flow_meter_def_policy *def_policy;
16713         struct mlx5_flow_tbl_resource *jump_tbl;
16714         struct mlx5_flow_tbl_data_entry *tbl_data;
16715         uint8_t egress, transfer;
16716         struct rte_flow_error error;
16717         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16718         int ret;
16719
16720         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16721         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16722         def_policy = mtrmng->def_policy[domain];
16723         if (!def_policy) {
16724                 def_policy = mlx5_malloc(MLX5_MEM_ZERO,
16725                         sizeof(struct mlx5_flow_meter_def_policy),
16726                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
16727                 if (!def_policy) {
16728                         DRV_LOG(ERR, "Failed to alloc default policy table.");
16729                         goto def_policy_error;
16730                 }
16731                 mtrmng->def_policy[domain] = def_policy;
16732                 /* Create the meter suffix table with SUFFIX level. */
16733                 jump_tbl = flow_dv_tbl_resource_get(dev,
16734                                 MLX5_FLOW_TABLE_LEVEL_METER,
16735                                 egress, transfer, false, NULL, 0,
16736                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16737                 if (!jump_tbl) {
16738                         DRV_LOG(ERR,
16739                                 "Failed to create meter suffix table.");
16740                         goto def_policy_error;
16741                 }
16742                 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
16743                 tbl_data = container_of(jump_tbl,
16744                                         struct mlx5_flow_tbl_data_entry, tbl);
16745                 def_policy->dr_jump_action[RTE_COLOR_GREEN] =
16746                                                 tbl_data->jump.action;
16747                 acts[RTE_COLOR_GREEN].dv_actions[0] = tbl_data->jump.action;
16748                 acts[RTE_COLOR_GREEN].actions_n = 1;
16749                 /*
16750                  * YELLOW has the same default policy as GREEN does.
16751                  * G & Y share the same table and action. The 2nd time of table
16752                  * resource getting is just to update the reference count for
16753                  * the releasing stage.
16754                  */
16755                 jump_tbl = flow_dv_tbl_resource_get(dev,
16756                                 MLX5_FLOW_TABLE_LEVEL_METER,
16757                                 egress, transfer, false, NULL, 0,
16758                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16759                 if (!jump_tbl) {
16760                         DRV_LOG(ERR,
16761                                 "Failed to get meter suffix table.");
16762                         goto def_policy_error;
16763                 }
16764                 def_policy->sub_policy.jump_tbl[RTE_COLOR_YELLOW] = jump_tbl;
16765                 tbl_data = container_of(jump_tbl,
16766                                         struct mlx5_flow_tbl_data_entry, tbl);
16767                 def_policy->dr_jump_action[RTE_COLOR_YELLOW] =
16768                                                 tbl_data->jump.action;
16769                 acts[RTE_COLOR_YELLOW].dv_actions[0] = tbl_data->jump.action;
16770                 acts[RTE_COLOR_YELLOW].actions_n = 1;
16771                 /* Create jump action to the drop table. */
16772                 if (!mtrmng->drop_tbl[domain]) {
16773                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
16774                                 (dev, MLX5_FLOW_TABLE_LEVEL_METER,
16775                                  egress, transfer, false, NULL, 0,
16776                                  0, MLX5_MTR_TABLE_ID_DROP, &error);
16777                         if (!mtrmng->drop_tbl[domain]) {
16778                                 DRV_LOG(ERR, "Failed to create meter "
16779                                         "drop table for default policy.");
16780                                 goto def_policy_error;
16781                         }
16782                 }
16783                 /* all RED: unique Drop table for jump action. */
16784                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16785                                         struct mlx5_flow_tbl_data_entry, tbl);
16786                 def_policy->dr_jump_action[RTE_COLOR_RED] =
16787                                                 tbl_data->jump.action;
16788                 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
16789                 acts[RTE_COLOR_RED].actions_n = 1;
16790                 /* Create default policy rules. */
16791                 ret = __flow_dv_create_domain_policy_rules(dev,
16792                                         &def_policy->sub_policy,
16793                                         egress, transfer, false, acts);
16794                 if (ret) {
16795                         DRV_LOG(ERR, "Failed to create default policy rules.");
16796                         goto def_policy_error;
16797                 }
16798         }
16799         return 0;
16800 def_policy_error:
16801         __flow_dv_destroy_domain_def_policy(dev,
16802                                             (enum mlx5_meter_domain)domain);
16803         return -1;
16804 }
16805
16806 /**
16807  * Create the default policy table set.
16808  *
16809  * @param[in] dev
16810  *   Pointer to Ethernet device.
16811  * @return
16812  *   0 on success, -1 otherwise.
16813  */
16814 static int
16815 flow_dv_create_def_policy(struct rte_eth_dev *dev)
16816 {
16817         struct mlx5_priv *priv = dev->data->dev_private;
16818         int i;
16819
16820         /* Non-termination policy table. */
16821         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16822                 if (!priv->sh->config.dv_esw_en &&
16823                     i == MLX5_MTR_DOMAIN_TRANSFER)
16824                         continue;
16825                 if (__flow_dv_create_domain_def_policy(dev, i)) {
16826                         DRV_LOG(ERR, "Failed to create default policy");
16827                         /* Rollback the created default policies for others. */
16828                         flow_dv_destroy_def_policy(dev);
16829                         return -1;
16830                 }
16831         }
16832         return 0;
16833 }
16834
16835 /**
16836  * Create the needed meter tables.
16837  * Lock free, (mutex should be acquired by caller).
16838  *
16839  * @param[in] dev
16840  *   Pointer to Ethernet device.
16841  * @param[in] fm
16842  *   Meter information table.
16843  * @param[in] mtr_idx
16844  *   Meter index.
16845  * @param[in] domain_bitmap
16846  *   Domain bitmap.
16847  * @return
16848  *   0 on success, -1 otherwise.
16849  */
16850 static int
16851 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
16852                         struct mlx5_flow_meter_info *fm,
16853                         uint32_t mtr_idx,
16854                         uint8_t domain_bitmap)
16855 {
16856         struct mlx5_priv *priv = dev->data->dev_private;
16857         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16858         struct rte_flow_error error;
16859         struct mlx5_flow_tbl_data_entry *tbl_data;
16860         uint8_t egress, transfer;
16861         void *actions[METER_ACTIONS];
16862         int domain, ret, i;
16863         struct mlx5_flow_counter *cnt;
16864         struct mlx5_flow_dv_match_params value = {
16865                 .size = sizeof(value.buf),
16866         };
16867         struct mlx5_flow_dv_match_params matcher_para = {
16868                 .size = sizeof(matcher_para.buf),
16869         };
16870         int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
16871                                                      0, &error);
16872         uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
16873         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
16874         struct mlx5_list_entry *entry;
16875         struct mlx5_flow_dv_matcher matcher = {
16876                 .mask = {
16877                         .size = sizeof(matcher.mask.buf),
16878                 },
16879         };
16880         struct mlx5_flow_dv_matcher *drop_matcher;
16881         struct mlx5_flow_cb_ctx ctx = {
16882                 .error = &error,
16883                 .data = &matcher,
16884         };
16885         uint8_t misc_mask;
16886
16887         if (!priv->mtr_en || mtr_id_reg_c < 0) {
16888                 rte_errno = ENOTSUP;
16889                 return -1;
16890         }
16891         for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
16892                 if (!(domain_bitmap & (1 << domain)) ||
16893                         (mtrmng->def_rule[domain] && !fm->drop_cnt))
16894                         continue;
16895                 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16896                 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16897                 /* Create the drop table with METER DROP level. */
16898                 if (!mtrmng->drop_tbl[domain]) {
16899                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
16900                                         MLX5_FLOW_TABLE_LEVEL_METER,
16901                                         egress, transfer, false, NULL, 0,
16902                                         0, MLX5_MTR_TABLE_ID_DROP, &error);
16903                         if (!mtrmng->drop_tbl[domain]) {
16904                                 DRV_LOG(ERR, "Failed to create meter drop table.");
16905                                 goto policy_error;
16906                         }
16907                 }
16908                 /* Create default matcher in drop table. */
16909                 matcher.tbl = mtrmng->drop_tbl[domain],
16910                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16911                                 struct mlx5_flow_tbl_data_entry, tbl);
16912                 if (!mtrmng->def_matcher[domain]) {
16913                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16914                                        (enum modify_reg)mtr_id_reg_c,
16915                                        0, 0);
16916                         matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
16917                         matcher.crc = rte_raw_cksum
16918                                         ((const void *)matcher.mask.buf,
16919                                         matcher.mask.size);
16920                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16921                         if (!entry) {
16922                                 DRV_LOG(ERR, "Failed to register meter "
16923                                 "drop default matcher.");
16924                                 goto policy_error;
16925                         }
16926                         mtrmng->def_matcher[domain] = container_of(entry,
16927                         struct mlx5_flow_dv_matcher, entry);
16928                 }
16929                 /* Create default rule in drop table. */
16930                 if (!mtrmng->def_rule[domain]) {
16931                         i = 0;
16932                         actions[i++] = priv->sh->dr_drop_action;
16933                         flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16934                                 (enum modify_reg)mtr_id_reg_c, 0, 0);
16935                         misc_mask = flow_dv_matcher_enable(value.buf);
16936                         __flow_dv_adjust_buf_size(&value.size, misc_mask);
16937                         ret = mlx5_flow_os_create_flow
16938                                 (mtrmng->def_matcher[domain]->matcher_object,
16939                                 (void *)&value, i, actions,
16940                                 &mtrmng->def_rule[domain]);
16941                         if (ret) {
16942                                 DRV_LOG(ERR, "Failed to create meter "
16943                                 "default drop rule for drop table.");
16944                                 goto policy_error;
16945                         }
16946                 }
16947                 if (!fm->drop_cnt)
16948                         continue;
16949                 MLX5_ASSERT(mtrmng->max_mtr_bits);
16950                 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
16951                         /* Create matchers for Drop. */
16952                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16953                                         (enum modify_reg)mtr_id_reg_c, 0,
16954                                         (mtr_id_mask << mtr_id_offset));
16955                         matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
16956                         matcher.crc = rte_raw_cksum
16957                                         ((const void *)matcher.mask.buf,
16958                                         matcher.mask.size);
16959                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16960                         if (!entry) {
16961                                 DRV_LOG(ERR,
16962                                 "Failed to register meter drop matcher.");
16963                                 goto policy_error;
16964                         }
16965                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
16966                                 container_of(entry, struct mlx5_flow_dv_matcher,
16967                                              entry);
16968                 }
16969                 drop_matcher =
16970                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
16971                 /* Create drop rule, matching meter_id only. */
16972                 flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16973                                 (enum modify_reg)mtr_id_reg_c,
16974                                 (mtr_idx << mtr_id_offset), UINT32_MAX);
16975                 i = 0;
16976                 cnt = flow_dv_counter_get_by_idx(dev,
16977                                         fm->drop_cnt, NULL);
16978                 actions[i++] = cnt->action;
16979                 actions[i++] = priv->sh->dr_drop_action;
16980                 misc_mask = flow_dv_matcher_enable(value.buf);
16981                 __flow_dv_adjust_buf_size(&value.size, misc_mask);
16982                 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
16983                                                (void *)&value, i, actions,
16984                                                &fm->drop_rule[domain]);
16985                 if (ret) {
16986                         DRV_LOG(ERR, "Failed to create meter "
16987                                 "drop rule for drop table.");
16988                                 goto policy_error;
16989                 }
16990         }
16991         return 0;
16992 policy_error:
16993         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16994                 if (fm->drop_rule[i]) {
16995                         claim_zero(mlx5_flow_os_destroy_flow
16996                                 (fm->drop_rule[i]));
16997                         fm->drop_rule[i] = NULL;
16998                 }
16999         }
17000         return -1;
17001 }
17002
17003 static struct mlx5_flow_meter_sub_policy *
17004 __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
17005                 struct mlx5_flow_meter_policy *mtr_policy,
17006                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS],
17007                 struct mlx5_flow_meter_sub_policy *next_sub_policy,
17008                 bool *is_reuse)
17009 {
17010         struct mlx5_priv *priv = dev->data->dev_private;
17011         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
17012         uint32_t sub_policy_idx = 0;
17013         uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
17014         uint32_t i, j;
17015         struct mlx5_hrxq *hrxq;
17016         struct mlx5_flow_handle dh;
17017         struct mlx5_meter_policy_action_container *act_cnt;
17018         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
17019         uint16_t sub_policy_num;
17020         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
17021
17022         MLX5_ASSERT(wks);
17023         rte_spinlock_lock(&mtr_policy->sl);
17024         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17025                 if (!rss_desc[i])
17026                         continue;
17027                 hrxq = mlx5_hrxq_get(dev, rss_desc[i]);
17028                 if (!hrxq) {
17029                         rte_spinlock_unlock(&mtr_policy->sl);
17030                         return NULL;
17031                 }
17032                 hrxq_idx[i] = hrxq->idx;
17033         }
17034         sub_policy_num = (mtr_policy->sub_policy_num >>
17035                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17036                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17037         for (j = 0; j < sub_policy_num; j++) {
17038                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17039                         if (rss_desc[i] &&
17040                             hrxq_idx[i] !=
17041                             mtr_policy->sub_policys[domain][j]->rix_hrxq[i])
17042                                 break;
17043                 }
17044                 if (i >= MLX5_MTR_RTE_COLORS) {
17045                         /*
17046                          * Found the sub policy table with
17047                          * the same queue per color.
17048                          */
17049                         rte_spinlock_unlock(&mtr_policy->sl);
17050                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
17051                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
17052                         *is_reuse = true;
17053                         return mtr_policy->sub_policys[domain][j];
17054                 }
17055         }
17056         /* Create sub policy. */
17057         if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
17058                 /* Reuse the first pre-allocated sub_policy. */
17059                 sub_policy = mtr_policy->sub_policys[domain][0];
17060                 sub_policy_idx = sub_policy->idx;
17061         } else {
17062                 sub_policy = mlx5_ipool_zmalloc
17063                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17064                                  &sub_policy_idx);
17065                 if (!sub_policy ||
17066                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
17067                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
17068                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
17069                         goto rss_sub_policy_error;
17070                 }
17071                 sub_policy->idx = sub_policy_idx;
17072                 sub_policy->main_policy = mtr_policy;
17073         }
17074         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17075                 if (!rss_desc[i])
17076                         continue;
17077                 sub_policy->rix_hrxq[i] = hrxq_idx[i];
17078                 if (mtr_policy->is_hierarchy) {
17079                         act_cnt = &mtr_policy->act_cnt[i];
17080                         act_cnt->next_sub_policy = next_sub_policy;
17081                         mlx5_hrxq_release(dev, hrxq_idx[i]);
17082                 } else {
17083                         /*
17084                          * Overwrite the last action from
17085                          * RSS action to Queue action.
17086                          */
17087                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
17088                                               hrxq_idx[i]);
17089                         if (!hrxq) {
17090                                 DRV_LOG(ERR, "Failed to get policy hrxq");
17091                                 goto rss_sub_policy_error;
17092                         }
17093                         act_cnt = &mtr_policy->act_cnt[i];
17094                         if (act_cnt->rix_mark || act_cnt->modify_hdr) {
17095                                 memset(&dh, 0, sizeof(struct mlx5_flow_handle));
17096                                 if (act_cnt->rix_mark)
17097                                         wks->mark = 1;
17098                                 dh.fate_action = MLX5_FLOW_FATE_QUEUE;
17099                                 dh.rix_hrxq = hrxq_idx[i];
17100                                 flow_drv_rxq_flags_set(dev, &dh);
17101                         }
17102                 }
17103         }
17104         if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
17105                                                sub_policy, domain)) {
17106                 DRV_LOG(ERR, "Failed to create policy "
17107                         "rules for ingress domain.");
17108                 goto rss_sub_policy_error;
17109         }
17110         if (sub_policy != mtr_policy->sub_policys[domain][0]) {
17111                 i = (mtr_policy->sub_policy_num >>
17112                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17113                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17114                 if (i >= MLX5_MTR_RSS_MAX_SUB_POLICY) {
17115                         DRV_LOG(ERR, "No free sub-policy slot.");
17116                         goto rss_sub_policy_error;
17117                 }
17118                 mtr_policy->sub_policys[domain][i] = sub_policy;
17119                 i++;
17120                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
17121                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
17122                 mtr_policy->sub_policy_num |=
17123                         (i & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
17124                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
17125         }
17126         rte_spinlock_unlock(&mtr_policy->sl);
17127         *is_reuse = false;
17128         return sub_policy;
17129 rss_sub_policy_error:
17130         if (sub_policy) {
17131                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
17132                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
17133                         i = (mtr_policy->sub_policy_num >>
17134                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17135                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17136                         mtr_policy->sub_policys[domain][i] = NULL;
17137                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17138                                         sub_policy->idx);
17139                 }
17140         }
17141         rte_spinlock_unlock(&mtr_policy->sl);
17142         return NULL;
17143 }
17144
17145 /**
17146  * Find the policy table for prefix table with RSS.
17147  *
17148  * @param[in] dev
17149  *   Pointer to Ethernet device.
17150  * @param[in] mtr_policy
17151  *   Pointer to meter policy table.
17152  * @param[in] rss_desc
17153  *   Pointer to rss_desc
17154  * @return
17155  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
17156  */
17157 static struct mlx5_flow_meter_sub_policy *
17158 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
17159                 struct mlx5_flow_meter_policy *mtr_policy,
17160                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
17161 {
17162         struct mlx5_priv *priv = dev->data->dev_private;
17163         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
17164         struct mlx5_flow_meter_info *next_fm;
17165         struct mlx5_flow_meter_policy *next_policy;
17166         struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL;
17167         struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM];
17168         struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM];
17169         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
17170         bool reuse_sub_policy;
17171         uint32_t i = 0;
17172         uint32_t j = 0;
17173
17174         while (true) {
17175                 /* Iterate hierarchy to get all policies in this hierarchy. */
17176                 policies[i++] = mtr_policy;
17177                 if (!mtr_policy->is_hierarchy)
17178                         break;
17179                 if (i >= MLX5_MTR_CHAIN_MAX_NUM) {
17180                         DRV_LOG(ERR, "Exceed max meter number in hierarchy.");
17181                         return NULL;
17182                 }
17183                 next_fm = mlx5_flow_meter_find(priv,
17184                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
17185                 if (!next_fm) {
17186                         DRV_LOG(ERR, "Failed to get next meter in hierarchy.");
17187                         return NULL;
17188                 }
17189                 next_policy =
17190                         mlx5_flow_meter_policy_find(dev, next_fm->policy_id,
17191                                                     NULL);
17192                 MLX5_ASSERT(next_policy);
17193                 mtr_policy = next_policy;
17194         }
17195         while (i) {
17196                 /**
17197                  * From last policy to the first one in hierarchy,
17198                  * create / get the sub policy for each of them.
17199                  */
17200                 sub_policy = __flow_dv_meter_get_rss_sub_policy(dev,
17201                                                         policies[--i],
17202                                                         rss_desc,
17203                                                         next_sub_policy,
17204                                                         &reuse_sub_policy);
17205                 if (!sub_policy) {
17206                         DRV_LOG(ERR, "Failed to get the sub policy.");
17207                         goto err_exit;
17208                 }
17209                 if (!reuse_sub_policy)
17210                         sub_policies[j++] = sub_policy;
17211                 next_sub_policy = sub_policy;
17212         }
17213         return sub_policy;
17214 err_exit:
17215         while (j) {
17216                 uint16_t sub_policy_num;
17217
17218                 sub_policy = sub_policies[--j];
17219                 mtr_policy = sub_policy->main_policy;
17220                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
17221                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
17222                         sub_policy_num = (mtr_policy->sub_policy_num >>
17223                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17224                                 MLX5_MTR_SUB_POLICY_NUM_MASK;
17225                         mtr_policy->sub_policys[domain][sub_policy_num - 1] =
17226                                                                         NULL;
17227                         sub_policy_num--;
17228                         mtr_policy->sub_policy_num &=
17229                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
17230                                   (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
17231                         mtr_policy->sub_policy_num |=
17232                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
17233                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
17234                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17235                                         sub_policy->idx);
17236                 }
17237         }
17238         return NULL;
17239 }
17240
17241 /**
17242  * Create the sub policy tag rule for all meters in hierarchy.
17243  *
17244  * @param[in] dev
17245  *   Pointer to Ethernet device.
17246  * @param[in] fm
17247  *   Meter information table.
17248  * @param[in] src_port
17249  *   The src port this extra rule should use.
17250  * @param[in] item
17251  *   The src port match item.
17252  * @param[out] error
17253  *   Perform verbose error reporting if not NULL.
17254  * @return
17255  *   0 on success, a negative errno value otherwise and rte_errno is set.
17256  */
17257 static int
17258 flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
17259                                 struct mlx5_flow_meter_info *fm,
17260                                 int32_t src_port,
17261                                 const struct rte_flow_item *item,
17262                                 struct rte_flow_error *error)
17263 {
17264         struct mlx5_priv *priv = dev->data->dev_private;
17265         struct mlx5_flow_meter_policy *mtr_policy;
17266         struct mlx5_flow_meter_sub_policy *sub_policy;
17267         struct mlx5_flow_meter_info *next_fm = NULL;
17268         struct mlx5_flow_meter_policy *next_policy;
17269         struct mlx5_flow_meter_sub_policy *next_sub_policy;
17270         struct mlx5_flow_tbl_data_entry *tbl_data;
17271         struct mlx5_sub_policy_color_rule *color_rule;
17272         struct mlx5_meter_policy_acts acts;
17273         uint32_t color_reg_c_idx;
17274         bool mtr_first = (src_port != UINT16_MAX) ? true : false;
17275         struct rte_flow_attr attr = {
17276                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
17277                 .priority = 0,
17278                 .ingress = 0,
17279                 .egress = 0,
17280                 .transfer = 1,
17281                 .reserved = 0,
17282         };
17283         uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
17284         int i;
17285
17286         mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
17287         MLX5_ASSERT(mtr_policy);
17288         if (!mtr_policy->is_hierarchy)
17289                 return 0;
17290         next_fm = mlx5_flow_meter_find(priv,
17291                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
17292         if (!next_fm) {
17293                 return rte_flow_error_set(error, EINVAL,
17294                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
17295                                 "Failed to find next meter in hierarchy.");
17296         }
17297         if (!next_fm->drop_cnt)
17298                 goto exit;
17299         color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error);
17300         sub_policy = mtr_policy->sub_policys[domain][0];
17301         for (i = 0; i < RTE_COLORS; i++) {
17302                 bool rule_exist = false;
17303                 struct mlx5_meter_policy_action_container *act_cnt;
17304
17305                 if (i >= RTE_COLOR_YELLOW)
17306                         break;
17307                 TAILQ_FOREACH(color_rule,
17308                               &sub_policy->color_rules[i], next_port)
17309                         if (color_rule->src_port == src_port) {
17310                                 rule_exist = true;
17311                                 break;
17312                         }
17313                 if (rule_exist)
17314                         continue;
17315                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
17316                                 sizeof(struct mlx5_sub_policy_color_rule),
17317                                 0, SOCKET_ID_ANY);
17318                 if (!color_rule)
17319                         return rte_flow_error_set(error, ENOMEM,
17320                                 RTE_FLOW_ERROR_TYPE_ACTION,
17321                                 NULL, "No memory to create tag color rule.");
17322                 color_rule->src_port = src_port;
17323                 attr.priority = i;
17324                 next_policy = mlx5_flow_meter_policy_find(dev,
17325                                                 next_fm->policy_id, NULL);
17326                 MLX5_ASSERT(next_policy);
17327                 next_sub_policy = next_policy->sub_policys[domain][0];
17328                 tbl_data = container_of(next_sub_policy->tbl_rsc,
17329                                         struct mlx5_flow_tbl_data_entry, tbl);
17330                 act_cnt = &mtr_policy->act_cnt[i];
17331                 if (mtr_first) {
17332                         acts.dv_actions[0] = next_fm->meter_action;
17333                         acts.dv_actions[1] = act_cnt->modify_hdr->action;
17334                 } else {
17335                         acts.dv_actions[0] = act_cnt->modify_hdr->action;
17336                         acts.dv_actions[1] = next_fm->meter_action;
17337                 }
17338                 acts.dv_actions[2] = tbl_data->jump.action;
17339                 acts.actions_n = 3;
17340                 if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) {
17341                         next_fm = NULL;
17342                         goto err_exit;
17343                 }
17344                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
17345                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
17346                                 &attr, true, item,
17347                                 &color_rule->matcher, error)) {
17348                         rte_flow_error_set(error, errno,
17349                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17350                                 "Failed to create hierarchy meter matcher.");
17351                         goto err_exit;
17352                 }
17353                 if (__flow_dv_create_policy_flow(dev, color_reg_c_idx,
17354                                         (enum rte_color)i,
17355                                         color_rule->matcher->matcher_object,
17356                                         acts.actions_n, acts.dv_actions,
17357                                         true, item,
17358                                         &color_rule->rule, &attr)) {
17359                         rte_flow_error_set(error, errno,
17360                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17361                                 "Failed to create hierarchy meter rule.");
17362                         goto err_exit;
17363                 }
17364                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
17365                                   color_rule, next_port);
17366         }
17367 exit:
17368         /**
17369          * Recursive call to iterate all meters in hierarchy and
17370          * create needed rules.
17371          */
17372         return flow_dv_meter_hierarchy_rule_create(dev, next_fm,
17373                                                 src_port, item, error);
17374 err_exit:
17375         if (color_rule) {
17376                 if (color_rule->rule)
17377                         mlx5_flow_os_destroy_flow(color_rule->rule);
17378                 if (color_rule->matcher) {
17379                         struct mlx5_flow_tbl_data_entry *tbl =
17380                                 container_of(color_rule->matcher->tbl,
17381                                                 typeof(*tbl), tbl);
17382                         mlx5_list_unregister(tbl->matchers,
17383                                                 &color_rule->matcher->entry);
17384                 }
17385                 mlx5_free(color_rule);
17386         }
17387         if (next_fm)
17388                 mlx5_flow_meter_detach(priv, next_fm);
17389         return -rte_errno;
17390 }
17391
17392 /**
17393  * Destroy the sub policy table with RX queue.
17394  *
17395  * @param[in] dev
17396  *   Pointer to Ethernet device.
17397  * @param[in] mtr_policy
17398  *   Pointer to meter policy table.
17399  */
17400 static void
17401 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
17402                                     struct mlx5_flow_meter_policy *mtr_policy)
17403 {
17404         struct mlx5_priv *priv = dev->data->dev_private;
17405         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
17406         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
17407         uint32_t i, j;
17408         uint16_t sub_policy_num, new_policy_num;
17409
17410         rte_spinlock_lock(&mtr_policy->sl);
17411         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17412                 switch (mtr_policy->act_cnt[i].fate_action) {
17413                 case MLX5_FLOW_FATE_SHARED_RSS:
17414                         sub_policy_num = (mtr_policy->sub_policy_num >>
17415                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17416                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17417                         new_policy_num = sub_policy_num;
17418                         for (j = 0; j < sub_policy_num; j++) {
17419                                 sub_policy =
17420                                         mtr_policy->sub_policys[domain][j];
17421                                 if (sub_policy) {
17422                                         __flow_dv_destroy_sub_policy_rules(dev,
17423                                                 sub_policy);
17424                                 if (sub_policy !=
17425                                         mtr_policy->sub_policys[domain][0]) {
17426                                         mtr_policy->sub_policys[domain][j] =
17427                                                                 NULL;
17428                                         mlx5_ipool_free
17429                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17430                                                 sub_policy->idx);
17431                                                 new_policy_num--;
17432                                         }
17433                                 }
17434                         }
17435                         if (new_policy_num != sub_policy_num) {
17436                                 mtr_policy->sub_policy_num &=
17437                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
17438                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
17439                                 mtr_policy->sub_policy_num |=
17440                                 (new_policy_num &
17441                                         MLX5_MTR_SUB_POLICY_NUM_MASK) <<
17442                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
17443                         }
17444                         break;
17445                 case MLX5_FLOW_FATE_QUEUE:
17446                         sub_policy = mtr_policy->sub_policys[domain][0];
17447                         __flow_dv_destroy_sub_policy_rules(dev,
17448                                                            sub_policy);
17449                         break;
17450                 default:
17451                         /*Other actions without queue and do nothing*/
17452                         break;
17453                 }
17454         }
17455         rte_spinlock_unlock(&mtr_policy->sl);
17456 }
17457 /**
17458  * Check whether the DR drop action is supported on the root table or not.
17459  *
17460  * Create a simple flow with DR drop action on root table to validate
17461  * if DR drop action on root table is supported or not.
17462  *
17463  * @param[in] dev
17464  *   Pointer to rte_eth_dev structure.
17465  *
17466  * @return
17467  *   0 on success, a negative errno value otherwise and rte_errno is set.
17468  */
17469 int
17470 mlx5_flow_discover_dr_action_support(struct rte_eth_dev *dev)
17471 {
17472         struct mlx5_priv *priv = dev->data->dev_private;
17473         struct mlx5_dev_ctx_shared *sh = priv->sh;
17474         struct mlx5_flow_dv_match_params mask = {
17475                 .size = sizeof(mask.buf),
17476         };
17477         struct mlx5_flow_dv_match_params value = {
17478                 .size = sizeof(value.buf),
17479         };
17480         struct mlx5dv_flow_matcher_attr dv_attr = {
17481                 .type = IBV_FLOW_ATTR_NORMAL,
17482                 .priority = 0,
17483                 .match_criteria_enable = 0,
17484                 .match_mask = (void *)&mask,
17485         };
17486         struct mlx5_flow_tbl_resource *tbl = NULL;
17487         void *matcher = NULL;
17488         void *flow = NULL;
17489         int ret = -1;
17490
17491         tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL,
17492                                         0, 0, 0, NULL);
17493         if (!tbl)
17494                 goto err;
17495         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
17496         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
17497         ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
17498                                                tbl->obj, &matcher);
17499         if (ret)
17500                 goto err;
17501         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
17502         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
17503                                        &sh->dr_drop_action, &flow);
17504 err:
17505         /*
17506          * If DR drop action is not supported on root table, flow create will
17507          * be failed with EOPNOTSUPP or EPROTONOSUPPORT.
17508          */
17509         if (!flow) {
17510                 if (matcher &&
17511                     (errno == EPROTONOSUPPORT || errno == EOPNOTSUPP))
17512                         DRV_LOG(INFO, "DR drop action is not supported in root table.");
17513                 else
17514                         DRV_LOG(ERR, "Unexpected error in DR drop action support detection");
17515                 ret = -1;
17516         } else {
17517                 claim_zero(mlx5_flow_os_destroy_flow(flow));
17518         }
17519         if (matcher)
17520                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
17521         if (tbl)
17522                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
17523         return ret;
17524 }
17525
17526 /**
17527  * Validate the batch counter support in root table.
17528  *
17529  * Create a simple flow with invalid counter and drop action on root table to
17530  * validate if batch counter with offset on root table is supported or not.
17531  *
17532  * @param[in] dev
17533  *   Pointer to rte_eth_dev structure.
17534  *
17535  * @return
17536  *   0 on success, a negative errno value otherwise and rte_errno is set.
17537  */
17538 int
17539 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
17540 {
17541         struct mlx5_priv *priv = dev->data->dev_private;
17542         struct mlx5_dev_ctx_shared *sh = priv->sh;
17543         struct mlx5_flow_dv_match_params mask = {
17544                 .size = sizeof(mask.buf),
17545         };
17546         struct mlx5_flow_dv_match_params value = {
17547                 .size = sizeof(value.buf),
17548         };
17549         struct mlx5dv_flow_matcher_attr dv_attr = {
17550                 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS,
17551                 .priority = 0,
17552                 .match_criteria_enable = 0,
17553                 .match_mask = (void *)&mask,
17554         };
17555         void *actions[2] = { 0 };
17556         struct mlx5_flow_tbl_resource *tbl = NULL;
17557         struct mlx5_devx_obj *dcs = NULL;
17558         void *matcher = NULL;
17559         void *flow = NULL;
17560         int ret = -1;
17561
17562         tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL,
17563                                         0, 0, 0, NULL);
17564         if (!tbl)
17565                 goto err;
17566         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0x4);
17567         if (!dcs)
17568                 goto err;
17569         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
17570                                                     &actions[0]);
17571         if (ret)
17572                 goto err;
17573         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
17574         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
17575         ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
17576                                                tbl->obj, &matcher);
17577         if (ret)
17578                 goto err;
17579         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
17580         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
17581                                        actions, &flow);
17582 err:
17583         /*
17584          * If batch counter with offset is not supported, the driver will not
17585          * validate the invalid offset value, flow create should success.
17586          * In this case, it means batch counter is not supported in root table.
17587          *
17588          * Otherwise, if flow create is failed, counter offset is supported.
17589          */
17590         if (flow) {
17591                 DRV_LOG(INFO, "Batch counter is not supported in root "
17592                               "table. Switch to fallback mode.");
17593                 rte_errno = ENOTSUP;
17594                 ret = -rte_errno;
17595                 claim_zero(mlx5_flow_os_destroy_flow(flow));
17596         } else {
17597                 /* Check matcher to make sure validate fail at flow create. */
17598                 if (!matcher || (matcher && errno != EINVAL))
17599                         DRV_LOG(ERR, "Unexpected error in counter offset "
17600                                      "support detection");
17601                 ret = 0;
17602         }
17603         if (actions[0])
17604                 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0]));
17605         if (matcher)
17606                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
17607         if (tbl)
17608                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
17609         if (dcs)
17610                 claim_zero(mlx5_devx_cmd_destroy(dcs));
17611         return ret;
17612 }
17613
17614 /**
17615  * Query a devx counter.
17616  *
17617  * @param[in] dev
17618  *   Pointer to the Ethernet device structure.
17619  * @param[in] cnt
17620  *   Index to the flow counter.
17621  * @param[in] clear
17622  *   Set to clear the counter statistics.
17623  * @param[out] pkts
17624  *   The statistics value of packets.
17625  * @param[out] bytes
17626  *   The statistics value of bytes.
17627  *
17628  * @return
17629  *   0 on success, otherwise return -1.
17630  */
17631 static int
17632 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
17633                       uint64_t *pkts, uint64_t *bytes, void **action)
17634 {
17635         struct mlx5_priv *priv = dev->data->dev_private;
17636         struct mlx5_flow_counter *cnt;
17637         uint64_t inn_pkts, inn_bytes;
17638         int ret;
17639
17640         if (!priv->sh->cdev->config.devx)
17641                 return -1;
17642
17643         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
17644         if (ret)
17645                 return -1;
17646         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
17647         if (cnt && action)
17648                 *action = cnt->action;
17649
17650         *pkts = inn_pkts - cnt->hits;
17651         *bytes = inn_bytes - cnt->bytes;
17652         if (clear) {
17653                 cnt->hits = inn_pkts;
17654                 cnt->bytes = inn_bytes;
17655         }
17656         return 0;
17657 }
17658
17659 /**
17660  * Get aged-out flows.
17661  *
17662  * @param[in] dev
17663  *   Pointer to the Ethernet device structure.
17664  * @param[in] context
17665  *   The address of an array of pointers to the aged-out flows contexts.
17666  * @param[in] nb_contexts
17667  *   The length of context array pointers.
17668  * @param[out] error
17669  *   Perform verbose error reporting if not NULL. Initialized in case of
17670  *   error only.
17671  *
17672  * @return
17673  *   how many contexts get in success, otherwise negative errno value.
17674  *   if nb_contexts is 0, return the amount of all aged contexts.
17675  *   if nb_contexts is not 0 , return the amount of aged flows reported
17676  *   in the context array.
17677  * @note: only stub for now
17678  */
17679 static int
17680 flow_dv_get_aged_flows(struct rte_eth_dev *dev,
17681                     void **context,
17682                     uint32_t nb_contexts,
17683                     struct rte_flow_error *error)
17684 {
17685         struct mlx5_priv *priv = dev->data->dev_private;
17686         struct mlx5_age_info *age_info;
17687         struct mlx5_age_param *age_param;
17688         struct mlx5_flow_counter *counter;
17689         struct mlx5_aso_age_action *act;
17690         int nb_flows = 0;
17691
17692         if (nb_contexts && !context)
17693                 return rte_flow_error_set(error, EINVAL,
17694                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17695                                           NULL, "empty context");
17696         age_info = GET_PORT_AGE_INFO(priv);
17697         rte_spinlock_lock(&age_info->aged_sl);
17698         LIST_FOREACH(act, &age_info->aged_aso, next) {
17699                 nb_flows++;
17700                 if (nb_contexts) {
17701                         context[nb_flows - 1] =
17702                                                 act->age_params.context;
17703                         if (!(--nb_contexts))
17704                                 break;
17705                 }
17706         }
17707         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
17708                 nb_flows++;
17709                 if (nb_contexts) {
17710                         age_param = MLX5_CNT_TO_AGE(counter);
17711                         context[nb_flows - 1] = age_param->context;
17712                         if (!(--nb_contexts))
17713                                 break;
17714                 }
17715         }
17716         rte_spinlock_unlock(&age_info->aged_sl);
17717         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
17718         return nb_flows;
17719 }
17720
17721 /*
17722  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
17723  */
17724 static uint32_t
17725 flow_dv_counter_allocate(struct rte_eth_dev *dev)
17726 {
17727         return flow_dv_counter_alloc(dev, 0);
17728 }
17729
17730 /**
17731  * Validate indirect action.
17732  * Dispatcher for action type specific validation.
17733  *
17734  * @param[in] dev
17735  *   Pointer to the Ethernet device structure.
17736  * @param[in] conf
17737  *   Indirect action configuration.
17738  * @param[in] action
17739  *   The indirect action object to validate.
17740  * @param[out] error
17741  *   Perform verbose error reporting if not NULL. Initialized in case of
17742  *   error only.
17743  *
17744  * @return
17745  *   0 on success, otherwise negative errno value.
17746  */
17747 int
17748 flow_dv_action_validate(struct rte_eth_dev *dev,
17749                         const struct rte_flow_indir_action_conf *conf,
17750                         const struct rte_flow_action *action,
17751                         struct rte_flow_error *err)
17752 {
17753         struct mlx5_priv *priv = dev->data->dev_private;
17754
17755         RTE_SET_USED(conf);
17756         switch (action->type) {
17757         case RTE_FLOW_ACTION_TYPE_RSS:
17758                 /*
17759                  * priv->obj_ops is set according to driver capabilities.
17760                  * When DevX capabilities are
17761                  * sufficient, it is set to devx_obj_ops.
17762                  * Otherwise, it is set to ibv_obj_ops.
17763                  * ibv_obj_ops doesn't support ind_table_modify operation.
17764                  * In this case the indirect RSS action can't be used.
17765                  */
17766                 if (priv->obj_ops.ind_table_modify == NULL)
17767                         return rte_flow_error_set
17768                                         (err, ENOTSUP,
17769                                          RTE_FLOW_ERROR_TYPE_ACTION,
17770                                          NULL,
17771                                          "Indirect RSS action not supported");
17772                 return mlx5_validate_action_rss(dev, action, err);
17773         case RTE_FLOW_ACTION_TYPE_AGE:
17774                 if (!priv->sh->aso_age_mng)
17775                         return rte_flow_error_set(err, ENOTSUP,
17776                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17777                                                 NULL,
17778                                                 "Indirect age action not supported");
17779                 return flow_dv_validate_action_age(0, action, dev, err);
17780         case RTE_FLOW_ACTION_TYPE_COUNT:
17781                 return flow_dv_validate_action_count(dev, true, 0, NULL, err);
17782         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
17783                 if (!priv->sh->ct_aso_en)
17784                         return rte_flow_error_set(err, ENOTSUP,
17785                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17786                                         "ASO CT is not supported");
17787                 return mlx5_validate_action_ct(dev, action->conf, err);
17788         default:
17789                 return rte_flow_error_set(err, ENOTSUP,
17790                                           RTE_FLOW_ERROR_TYPE_ACTION,
17791                                           NULL,
17792                                           "action type not supported");
17793         }
17794 }
17795
17796 /*
17797  * Check if the RSS configurations for colors of a meter policy match
17798  * each other, except the queues.
17799  *
17800  * @param[in] r1
17801  *   Pointer to the first RSS flow action.
17802  * @param[in] r2
17803  *   Pointer to the second RSS flow action.
17804  *
17805  * @return
17806  *   0 on match, 1 on conflict.
17807  */
17808 static inline int
17809 flow_dv_mtr_policy_rss_compare(const struct rte_flow_action_rss *r1,
17810                                const struct rte_flow_action_rss *r2)
17811 {
17812         if (r1 == NULL || r2 == NULL)
17813                 return 0;
17814         if (!(r1->level <= 1 && r2->level <= 1) &&
17815             !(r1->level > 1 && r2->level > 1))
17816                 return 1;
17817         if (r1->types != r2->types &&
17818             !((r1->types == 0 || r1->types == RTE_ETH_RSS_IP) &&
17819               (r2->types == 0 || r2->types == RTE_ETH_RSS_IP)))
17820                 return 1;
17821         if (r1->key || r2->key) {
17822                 const void *key1 = r1->key ? r1->key : rss_hash_default_key;
17823                 const void *key2 = r2->key ? r2->key : rss_hash_default_key;
17824
17825                 if (memcmp(key1, key2, MLX5_RSS_HASH_KEY_LEN))
17826                         return 1;
17827         }
17828         return 0;
17829 }
17830
17831 /**
17832  * Validate the meter hierarchy chain for meter policy.
17833  *
17834  * @param[in] dev
17835  *   Pointer to the Ethernet device structure.
17836  * @param[in] meter_id
17837  *   Meter id.
17838  * @param[in] action_flags
17839  *   Holds the actions detected until now.
17840  * @param[out] is_rss
17841  *   Is RSS or not.
17842  * @param[out] hierarchy_domain
17843  *   The domain bitmap for hierarchy policy.
17844  * @param[out] error
17845  *   Perform verbose error reporting if not NULL. Initialized in case of
17846  *   error only.
17847  *
17848  * @return
17849  *   0 on success, otherwise negative errno value with error set.
17850  */
17851 static int
17852 flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
17853                                   uint32_t meter_id,
17854                                   uint64_t action_flags,
17855                                   bool *is_rss,
17856                                   uint8_t *hierarchy_domain,
17857                                   struct rte_mtr_error *error)
17858 {
17859         struct mlx5_priv *priv = dev->data->dev_private;
17860         struct mlx5_flow_meter_info *fm;
17861         struct mlx5_flow_meter_policy *policy;
17862         uint8_t cnt = 1;
17863
17864         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
17865                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17866                 return -rte_mtr_error_set(error, EINVAL,
17867                                         RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN,
17868                                         NULL,
17869                                         "Multiple fate actions not supported.");
17870         *hierarchy_domain = 0;
17871         while (true) {
17872                 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
17873                 if (!fm)
17874                         return -rte_mtr_error_set(error, EINVAL,
17875                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17876                                         "Meter not found in meter hierarchy.");
17877                 if (fm->def_policy)
17878                         return -rte_mtr_error_set(error, EINVAL,
17879                                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17880                         "Non termination meter not supported in hierarchy.");
17881                 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
17882                 MLX5_ASSERT(policy);
17883                 /**
17884                  * Only inherit the supported domains of the first meter in
17885                  * hierarchy.
17886                  * One meter supports at least one domain.
17887                  */
17888                 if (!*hierarchy_domain) {
17889                         if (policy->transfer)
17890                                 *hierarchy_domain |=
17891                                                 MLX5_MTR_DOMAIN_TRANSFER_BIT;
17892                         if (policy->ingress)
17893                                 *hierarchy_domain |=
17894                                                 MLX5_MTR_DOMAIN_INGRESS_BIT;
17895                         if (policy->egress)
17896                                 *hierarchy_domain |= MLX5_MTR_DOMAIN_EGRESS_BIT;
17897                 }
17898                 if (!policy->is_hierarchy) {
17899                         *is_rss = policy->is_rss;
17900                         break;
17901                 }
17902                 meter_id = policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id;
17903                 if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM)
17904                         return -rte_mtr_error_set(error, EINVAL,
17905                                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
17906                                         "Exceed max hierarchy meter number.");
17907         }
17908         return 0;
17909 }
17910
17911 /**
17912  * Validate meter policy actions.
17913  * Dispatcher for action type specific validation.
17914  *
17915  * @param[in] dev
17916  *   Pointer to the Ethernet device structure.
17917  * @param[in] action
17918  *   The meter policy action object to validate.
17919  * @param[in] attr
17920  *   Attributes of flow to determine steering domain.
17921  * @param[out] error
17922  *   Perform verbose error reporting if not NULL. Initialized in case of
17923  *   error only.
17924  *
17925  * @return
17926  *   0 on success, otherwise negative errno value.
17927  */
17928 static int
17929 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
17930                         const struct rte_flow_action *actions[RTE_COLORS],
17931                         struct rte_flow_attr *attr,
17932                         bool *is_rss,
17933                         uint8_t *domain_bitmap,
17934                         uint8_t *policy_mode,
17935                         struct rte_mtr_error *error)
17936 {
17937         struct mlx5_priv *priv = dev->data->dev_private;
17938         struct mlx5_sh_config *dev_conf = &priv->sh->config;
17939         const struct rte_flow_action *act;
17940         uint64_t action_flags[RTE_COLORS] = {0};
17941         int actions_n;
17942         int i, ret;
17943         struct rte_flow_error flow_err;
17944         uint8_t domain_color[RTE_COLORS] = {0};
17945         uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
17946         uint8_t hierarchy_domain = 0;
17947         const struct rte_flow_action_meter *mtr;
17948         bool def_green = false;
17949         bool def_yellow = false;
17950         const struct rte_flow_action_rss *rss_color[RTE_COLORS] = {NULL};
17951
17952         if (!dev_conf->dv_esw_en)
17953                 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
17954         *domain_bitmap = def_domain;
17955         /* Red color could only support DROP action. */
17956         if (!actions[RTE_COLOR_RED] ||
17957             actions[RTE_COLOR_RED]->type != RTE_FLOW_ACTION_TYPE_DROP)
17958                 return -rte_mtr_error_set(error, ENOTSUP,
17959                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17960                                 NULL, "Red color only supports drop action.");
17961         /*
17962          * Check default policy actions:
17963          * Green / Yellow: no action, Red: drop action
17964          * Either G or Y will trigger default policy actions to be created.
17965          */
17966         if (!actions[RTE_COLOR_GREEN] ||
17967             actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END)
17968                 def_green = true;
17969         if (!actions[RTE_COLOR_YELLOW] ||
17970             actions[RTE_COLOR_YELLOW]->type == RTE_FLOW_ACTION_TYPE_END)
17971                 def_yellow = true;
17972         if (def_green && def_yellow) {
17973                 *policy_mode = MLX5_MTR_POLICY_MODE_DEF;
17974                 return 0;
17975         } else if (!def_green && def_yellow) {
17976                 *policy_mode = MLX5_MTR_POLICY_MODE_OG;
17977         } else if (def_green && !def_yellow) {
17978                 *policy_mode = MLX5_MTR_POLICY_MODE_OY;
17979         } else {
17980                 *policy_mode = MLX5_MTR_POLICY_MODE_ALL;
17981         }
17982         /* Set to empty string in case of NULL pointer access by user. */
17983         flow_err.message = "";
17984         for (i = 0; i < RTE_COLORS; i++) {
17985                 act = actions[i];
17986                 for (action_flags[i] = 0, actions_n = 0;
17987                      act && act->type != RTE_FLOW_ACTION_TYPE_END;
17988                      act++) {
17989                         if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
17990                                 return -rte_mtr_error_set(error, ENOTSUP,
17991                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17992                                           NULL, "too many actions");
17993                         switch (act->type) {
17994                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
17995                         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
17996                                 if (!dev_conf->dv_esw_en)
17997                                         return -rte_mtr_error_set(error,
17998                                         ENOTSUP,
17999                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18000                                         NULL, "PORT action validate check"
18001                                         " fail for ESW disable");
18002                                 ret = flow_dv_validate_action_port_id(dev,
18003                                                 action_flags[i],
18004                                                 act, attr, &flow_err);
18005                                 if (ret)
18006                                         return -rte_mtr_error_set(error,
18007                                         ENOTSUP,
18008                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18009                                         NULL, flow_err.message ?
18010                                         flow_err.message :
18011                                         "PORT action validate check fail");
18012                                 ++actions_n;
18013                                 action_flags[i] |= MLX5_FLOW_ACTION_PORT_ID;
18014                                 break;
18015                         case RTE_FLOW_ACTION_TYPE_MARK:
18016                                 ret = flow_dv_validate_action_mark(dev, act,
18017                                                            action_flags[i],
18018                                                            attr, &flow_err);
18019                                 if (ret < 0)
18020                                         return -rte_mtr_error_set(error,
18021                                         ENOTSUP,
18022                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18023                                         NULL, flow_err.message ?
18024                                         flow_err.message :
18025                                         "Mark action validate check fail");
18026                                 if (dev_conf->dv_xmeta_en !=
18027                                         MLX5_XMETA_MODE_LEGACY)
18028                                         return -rte_mtr_error_set(error,
18029                                         ENOTSUP,
18030                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18031                                         NULL, "Extend MARK action is "
18032                                         "not supported. Please try use "
18033                                         "default policy for meter.");
18034                                 action_flags[i] |= MLX5_FLOW_ACTION_MARK;
18035                                 ++actions_n;
18036                                 break;
18037                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
18038                                 ret = flow_dv_validate_action_set_tag(dev,
18039                                                         act, action_flags[i],
18040                                                         attr, &flow_err);
18041                                 if (ret)
18042                                         return -rte_mtr_error_set(error,
18043                                         ENOTSUP,
18044                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18045                                         NULL, flow_err.message ?
18046                                         flow_err.message :
18047                                         "Set tag action validate check fail");
18048                                 action_flags[i] |= MLX5_FLOW_ACTION_SET_TAG;
18049                                 ++actions_n;
18050                                 break;
18051                         case RTE_FLOW_ACTION_TYPE_DROP:
18052                                 ret = mlx5_flow_validate_action_drop
18053                                         (action_flags[i], attr, &flow_err);
18054                                 if (ret < 0)
18055                                         return -rte_mtr_error_set(error,
18056                                         ENOTSUP,
18057                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18058                                         NULL, flow_err.message ?
18059                                         flow_err.message :
18060                                         "Drop action validate check fail");
18061                                 action_flags[i] |= MLX5_FLOW_ACTION_DROP;
18062                                 ++actions_n;
18063                                 break;
18064                         case RTE_FLOW_ACTION_TYPE_QUEUE:
18065                                 /*
18066                                  * Check whether extensive
18067                                  * metadata feature is engaged.
18068                                  */
18069                                 if (dev_conf->dv_flow_en &&
18070                                     (dev_conf->dv_xmeta_en !=
18071                                      MLX5_XMETA_MODE_LEGACY) &&
18072                                     mlx5_flow_ext_mreg_supported(dev))
18073                                         return -rte_mtr_error_set(error,
18074                                           ENOTSUP,
18075                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18076                                           NULL, "Queue action with meta "
18077                                           "is not supported. Please try use "
18078                                           "default policy for meter.");
18079                                 ret = mlx5_flow_validate_action_queue(act,
18080                                                         action_flags[i], dev,
18081                                                         attr, &flow_err);
18082                                 if (ret < 0)
18083                                         return -rte_mtr_error_set(error,
18084                                           ENOTSUP,
18085                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18086                                           NULL, flow_err.message ?
18087                                           flow_err.message :
18088                                           "Queue action validate check fail");
18089                                 action_flags[i] |= MLX5_FLOW_ACTION_QUEUE;
18090                                 ++actions_n;
18091                                 break;
18092                         case RTE_FLOW_ACTION_TYPE_RSS:
18093                                 if (dev_conf->dv_flow_en &&
18094                                     (dev_conf->dv_xmeta_en !=
18095                                      MLX5_XMETA_MODE_LEGACY) &&
18096                                     mlx5_flow_ext_mreg_supported(dev))
18097                                         return -rte_mtr_error_set(error,
18098                                           ENOTSUP,
18099                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18100                                           NULL, "RSS action with meta "
18101                                           "is not supported. Please try use "
18102                                           "default policy for meter.");
18103                                 ret = mlx5_validate_action_rss(dev, act,
18104                                                                &flow_err);
18105                                 if (ret < 0)
18106                                         return -rte_mtr_error_set(error,
18107                                           ENOTSUP,
18108                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18109                                           NULL, flow_err.message ?
18110                                           flow_err.message :
18111                                           "RSS action validate check fail");
18112                                 action_flags[i] |= MLX5_FLOW_ACTION_RSS;
18113                                 ++actions_n;
18114                                 /* Either G or Y will set the RSS. */
18115                                 rss_color[i] = act->conf;
18116                                 break;
18117                         case RTE_FLOW_ACTION_TYPE_JUMP:
18118                                 ret = flow_dv_validate_action_jump(dev,
18119                                         NULL, act, action_flags[i],
18120                                         attr, true, &flow_err);
18121                                 if (ret)
18122                                         return -rte_mtr_error_set(error,
18123                                           ENOTSUP,
18124                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18125                                           NULL, flow_err.message ?
18126                                           flow_err.message :
18127                                           "Jump action validate check fail");
18128                                 ++actions_n;
18129                                 action_flags[i] |= MLX5_FLOW_ACTION_JUMP;
18130                                 break;
18131                         /*
18132                          * Only the last meter in the hierarchy will support
18133                          * the YELLOW color steering. Then in the meter policy
18134                          * actions list, there should be no other meter inside.
18135                          */
18136                         case RTE_FLOW_ACTION_TYPE_METER:
18137                                 if (i != RTE_COLOR_GREEN)
18138                                         return -rte_mtr_error_set(error,
18139                                                 ENOTSUP,
18140                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
18141                                                 NULL,
18142                                                 "Meter hierarchy only supports GREEN color.");
18143                                 if (*policy_mode != MLX5_MTR_POLICY_MODE_OG)
18144                                         return -rte_mtr_error_set(error,
18145                                                 ENOTSUP,
18146                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
18147                                                 NULL,
18148                                                 "No yellow policy should be provided in meter hierarchy.");
18149                                 mtr = act->conf;
18150                                 ret = flow_dv_validate_policy_mtr_hierarchy(dev,
18151                                                         mtr->mtr_id,
18152                                                         action_flags[i],
18153                                                         is_rss,
18154                                                         &hierarchy_domain,
18155                                                         error);
18156                                 if (ret)
18157                                         return ret;
18158                                 ++actions_n;
18159                                 action_flags[i] |=
18160                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
18161                                 break;
18162                         default:
18163                                 return -rte_mtr_error_set(error, ENOTSUP,
18164                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18165                                         NULL,
18166                                         "Doesn't support optional action");
18167                         }
18168                 }
18169                 if (action_flags[i] & MLX5_FLOW_ACTION_PORT_ID) {
18170                         domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT;
18171                 } else if ((action_flags[i] &
18172                           (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) ||
18173                           (action_flags[i] & MLX5_FLOW_ACTION_MARK)) {
18174                         /*
18175                          * Only support MLX5_XMETA_MODE_LEGACY
18176                          * so MARK action is only in ingress domain.
18177                          */
18178                         domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT;
18179                 } else {
18180                         domain_color[i] = def_domain;
18181                         if (action_flags[i] &&
18182                             !(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
18183                                 domain_color[i] &=
18184                                 ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
18185                 }
18186                 if (action_flags[i] &
18187                     MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
18188                         domain_color[i] &= hierarchy_domain;
18189                 /*
18190                  * Non-termination actions only support NIC Tx domain.
18191                  * The adjustion should be skipped when there is no
18192                  * action or only END is provided. The default domains
18193                  * bit-mask is set to find the MIN intersection.
18194                  * The action flags checking should also be skipped.
18195                  */
18196                 if ((def_green && i == RTE_COLOR_GREEN) ||
18197                     (def_yellow && i == RTE_COLOR_YELLOW))
18198                         continue;
18199                 /*
18200                  * Validate the drop action mutual exclusion
18201                  * with other actions. Drop action is mutually-exclusive
18202                  * with any other action, except for Count action.
18203                  */
18204                 if ((action_flags[i] & MLX5_FLOW_ACTION_DROP) &&
18205                     (action_flags[i] & ~MLX5_FLOW_ACTION_DROP)) {
18206                         return -rte_mtr_error_set(error, ENOTSUP,
18207                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
18208                                 NULL, "Drop action is mutually-exclusive "
18209                                 "with any other action");
18210                 }
18211                 /* Eswitch has few restrictions on using items and actions */
18212                 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) {
18213                         if (!mlx5_flow_ext_mreg_supported(dev) &&
18214                             action_flags[i] & MLX5_FLOW_ACTION_MARK)
18215                                 return -rte_mtr_error_set(error, ENOTSUP,
18216                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18217                                         NULL, "unsupported action MARK");
18218                         if (action_flags[i] & MLX5_FLOW_ACTION_QUEUE)
18219                                 return -rte_mtr_error_set(error, ENOTSUP,
18220                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18221                                         NULL, "unsupported action QUEUE");
18222                         if (action_flags[i] & MLX5_FLOW_ACTION_RSS)
18223                                 return -rte_mtr_error_set(error, ENOTSUP,
18224                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18225                                         NULL, "unsupported action RSS");
18226                         if (!(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
18227                                 return -rte_mtr_error_set(error, ENOTSUP,
18228                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18229                                         NULL, "no fate action is found");
18230                 } else {
18231                         if (!(action_flags[i] & MLX5_FLOW_FATE_ACTIONS) &&
18232                             (domain_color[i] & MLX5_MTR_DOMAIN_INGRESS_BIT)) {
18233                                 if ((domain_color[i] &
18234                                      MLX5_MTR_DOMAIN_EGRESS_BIT))
18235                                         domain_color[i] =
18236                                                 MLX5_MTR_DOMAIN_EGRESS_BIT;
18237                                 else
18238                                         return -rte_mtr_error_set(error,
18239                                                 ENOTSUP,
18240                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
18241                                                 NULL,
18242                                                 "no fate action is found");
18243                         }
18244                 }
18245         }
18246         /* If both colors have RSS, the attributes should be the same. */
18247         if (flow_dv_mtr_policy_rss_compare(rss_color[RTE_COLOR_GREEN],
18248                                            rss_color[RTE_COLOR_YELLOW]))
18249                 return -rte_mtr_error_set(error, EINVAL,
18250                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18251                                           NULL, "policy RSS attr conflict");
18252         if (rss_color[RTE_COLOR_GREEN] || rss_color[RTE_COLOR_YELLOW])
18253                 *is_rss = true;
18254         /* "domain_color[C]" is non-zero for each color, default is ALL. */
18255         if (!def_green && !def_yellow &&
18256             domain_color[RTE_COLOR_GREEN] != domain_color[RTE_COLOR_YELLOW] &&
18257             !(action_flags[RTE_COLOR_GREEN] & MLX5_FLOW_ACTION_DROP) &&
18258             !(action_flags[RTE_COLOR_YELLOW] & MLX5_FLOW_ACTION_DROP))
18259                 return -rte_mtr_error_set(error, EINVAL,
18260                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18261                                           NULL, "policy domains conflict");
18262         /*
18263          * At least one color policy is listed in the actions, the domains
18264          * to be supported should be the intersection.
18265          */
18266         *domain_bitmap = domain_color[RTE_COLOR_GREEN] &
18267                          domain_color[RTE_COLOR_YELLOW];
18268         return 0;
18269 }
18270
18271 static int
18272 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
18273 {
18274         struct mlx5_priv *priv = dev->data->dev_private;
18275         int ret = 0;
18276
18277         if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
18278                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain,
18279                                                 flags);
18280                 if (ret != 0)
18281                         return ret;
18282         }
18283         if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
18284                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags);
18285                 if (ret != 0)
18286                         return ret;
18287         }
18288         if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
18289                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags);
18290                 if (ret != 0)
18291                         return ret;
18292         }
18293         return 0;
18294 }
18295
18296 /**
18297  * Discover the number of available flow priorities
18298  * by trying to create a flow with the highest priority value
18299  * for each possible number.
18300  *
18301  * @param[in] dev
18302  *   Ethernet device.
18303  * @param[in] vprio
18304  *   List of possible number of available priorities.
18305  * @param[in] vprio_n
18306  *   Size of @p vprio array.
18307  * @return
18308  *   On success, number of available flow priorities.
18309  *   On failure, a negative errno-style code and rte_errno is set.
18310  */
18311 static int
18312 flow_dv_discover_priorities(struct rte_eth_dev *dev,
18313                             const uint16_t *vprio, int vprio_n)
18314 {
18315         struct mlx5_priv *priv = dev->data->dev_private;
18316         struct mlx5_indexed_pool *pool = priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW];
18317         struct rte_flow_item_eth eth;
18318         struct rte_flow_item item = {
18319                 .type = RTE_FLOW_ITEM_TYPE_ETH,
18320                 .spec = &eth,
18321                 .mask = &eth,
18322         };
18323         struct mlx5_flow_dv_matcher matcher = {
18324                 .mask = {
18325                         .size = sizeof(matcher.mask.buf),
18326                 },
18327         };
18328         union mlx5_flow_tbl_key tbl_key;
18329         struct mlx5_flow flow;
18330         void *action;
18331         struct rte_flow_error error;
18332         uint8_t misc_mask;
18333         int i, err, ret = -ENOTSUP;
18334
18335         /*
18336          * Prepare a flow with a catch-all pattern and a drop action.
18337          * Use drop queue, because shared drop action may be unavailable.
18338          */
18339         action = priv->drop_queue.hrxq->action;
18340         if (action == NULL) {
18341                 DRV_LOG(ERR, "Priority discovery requires a drop action");
18342                 rte_errno = ENOTSUP;
18343                 return -rte_errno;
18344         }
18345         memset(&flow, 0, sizeof(flow));
18346         flow.handle = mlx5_ipool_zmalloc(pool, &flow.handle_idx);
18347         if (flow.handle == NULL) {
18348                 DRV_LOG(ERR, "Cannot create flow handle");
18349                 rte_errno = ENOMEM;
18350                 return -rte_errno;
18351         }
18352         flow.ingress = true;
18353         flow.dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
18354         flow.dv.actions[0] = action;
18355         flow.dv.actions_n = 1;
18356         memset(&eth, 0, sizeof(eth));
18357         flow_dv_translate_item_eth(matcher.mask.buf, flow.dv.value.buf,
18358                                    &item, /* inner */ false, /* group */ 0);
18359         matcher.crc = rte_raw_cksum(matcher.mask.buf, matcher.mask.size);
18360         for (i = 0; i < vprio_n; i++) {
18361                 /* Configure the next proposed maximum priority. */
18362                 matcher.priority = vprio[i] - 1;
18363                 memset(&tbl_key, 0, sizeof(tbl_key));
18364                 err = flow_dv_matcher_register(dev, &matcher, &tbl_key, &flow,
18365                                                /* tunnel */ NULL,
18366                                                /* group */ 0,
18367                                                &error);
18368                 if (err != 0) {
18369                         /* This action is pure SW and must always succeed. */
18370                         DRV_LOG(ERR, "Cannot register matcher");
18371                         ret = -rte_errno;
18372                         break;
18373                 }
18374                 /* Try to apply the flow to HW. */
18375                 misc_mask = flow_dv_matcher_enable(flow.dv.value.buf);
18376                 __flow_dv_adjust_buf_size(&flow.dv.value.size, misc_mask);
18377                 err = mlx5_flow_os_create_flow
18378                                 (flow.handle->dvh.matcher->matcher_object,
18379                                  (void *)&flow.dv.value, flow.dv.actions_n,
18380                                  flow.dv.actions, &flow.handle->drv_flow);
18381                 if (err == 0) {
18382                         claim_zero(mlx5_flow_os_destroy_flow
18383                                                 (flow.handle->drv_flow));
18384                         flow.handle->drv_flow = NULL;
18385                 }
18386                 claim_zero(flow_dv_matcher_release(dev, flow.handle));
18387                 if (err != 0)
18388                         break;
18389                 ret = vprio[i];
18390         }
18391         mlx5_ipool_free(pool, flow.handle_idx);
18392         /* Set rte_errno if no expected priority value matched. */
18393         if (ret < 0)
18394                 rte_errno = -ret;
18395         return ret;
18396 }
18397
18398 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
18399         .validate = flow_dv_validate,
18400         .prepare = flow_dv_prepare,
18401         .translate = flow_dv_translate,
18402         .apply = flow_dv_apply,
18403         .remove = flow_dv_remove,
18404         .destroy = flow_dv_destroy,
18405         .query = flow_dv_query,
18406         .create_mtr_tbls = flow_dv_create_mtr_tbls,
18407         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls,
18408         .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls,
18409         .create_meter = flow_dv_mtr_alloc,
18410         .free_meter = flow_dv_aso_mtr_release_to_pool,
18411         .validate_mtr_acts = flow_dv_validate_mtr_policy_acts,
18412         .create_mtr_acts = flow_dv_create_mtr_policy_acts,
18413         .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts,
18414         .create_policy_rules = flow_dv_create_policy_rules,
18415         .destroy_policy_rules = flow_dv_destroy_policy_rules,
18416         .create_def_policy = flow_dv_create_def_policy,
18417         .destroy_def_policy = flow_dv_destroy_def_policy,
18418         .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare,
18419         .meter_hierarchy_rule_create = flow_dv_meter_hierarchy_rule_create,
18420         .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq,
18421         .counter_alloc = flow_dv_counter_allocate,
18422         .counter_free = flow_dv_counter_free,
18423         .counter_query = flow_dv_counter_query,
18424         .get_aged_flows = flow_dv_get_aged_flows,
18425         .action_validate = flow_dv_action_validate,
18426         .action_create = flow_dv_action_create,
18427         .action_destroy = flow_dv_action_destroy,
18428         .action_update = flow_dv_action_update,
18429         .action_query = flow_dv_action_query,
18430         .sync_domain = flow_dv_sync_domain,
18431         .discover_priorities = flow_dv_discover_priorities,
18432         .item_create = flow_dv_item_create,
18433         .item_release = flow_dv_item_release,
18434 };
18435
18436 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */