drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24 #include <rte_mtr.h>
  25 #include <rte_mtr_driver.h>
  26 #include <rte_tailq.h>
  27
  28 #include <mlx5_glue.h>
  29 #include <mlx5_devx_cmds.h>
  30 #include <mlx5_prm.h>
  31 #include <mlx5_malloc.h>
  32
  33 #include "mlx5_defs.h"
  34 #include "mlx5.h"
  35 #include "mlx5_common_os.h"
  36 #include "mlx5_flow.h"
  37 #include "mlx5_flow_os.h"
  38 #include "mlx5_rx.h"
  39 #include "mlx5_tx.h"
  40 #include "rte_pmd_mlx5.h"
  41
  42 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  43
  44 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  45 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  46 #endif
  47
  48 #ifndef HAVE_MLX5DV_DR_ESWITCH
  49 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  50 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  51 #endif
  52 #endif
  53
  54 #ifndef HAVE_MLX5DV_DR
  55 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  56 #endif
  57
  58 /* VLAN header definitions */
  59 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  60 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  61 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  62 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  63 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  64
  65 union flow_dv_attr {
  66         struct {
  67                 uint32_t valid:1;
  68                 uint32_t ipv4:1;
  69                 uint32_t ipv6:1;
  70                 uint32_t tcp:1;
  71                 uint32_t udp:1;
  72                 uint32_t reserved:27;
  73         };
  74         uint32_t attr;
  75 };
  76
  77 static int
  78 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
  79                              struct mlx5_flow_tbl_resource *tbl);
  80
  81 static int
  82 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  83                                      uint32_t encap_decap_idx);
  84
  85 static int
  86 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  87                                         uint32_t port_id);
  88 static void
  89 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);
  90
  91 static int
  92 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
  93                                   uint32_t rix_jump);
  94
  95 /**
  96  * Initialize flow attributes structure according to flow items' types.
  97  *
  98  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  99  * mode. For tunnel mode, the items to be modified are the outermost ones.
 100  *
 101  * @param[in] item
 102  *   Pointer to item specification.
 103  * @param[out] attr
 104  *   Pointer to flow attributes structure.
 105  * @param[in] dev_flow
 106  *   Pointer to the sub flow.
 107  * @param[in] tunnel_decap
 108  *   Whether action is after tunnel decapsulation.
 109  */
 110 static void
 111 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 112                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 113 {
 114         uint64_t layers = dev_flow->handle->layers;
 115
 116         /*
 117          * If layers is already initialized, it means this dev_flow is the
 118          * suffix flow, the layers flags is set by the prefix flow. Need to
 119          * use the layer flags from prefix flow as the suffix flow may not
 120          * have the user defined items as the flow is split.
 121          */
 122         if (layers) {
 123                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 124                         attr->ipv4 = 1;
 125                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 126                         attr->ipv6 = 1;
 127                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 128                         attr->tcp = 1;
 129                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 130                         attr->udp = 1;
 131                 attr->valid = 1;
 132                 return;
 133         }
 134         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 135                 uint8_t next_protocol = 0xff;
 136                 switch (item->type) {
 137                 case RTE_FLOW_ITEM_TYPE_GRE:
 138                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 139                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 140                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 141                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 142                 case RTE_FLOW_ITEM_TYPE_MPLS:
 143                         if (tunnel_decap)
 144                                 attr->attr = 0;
 145                         break;
 146                 case RTE_FLOW_ITEM_TYPE_IPV4:
 147                         if (!attr->ipv6)
 148                                 attr->ipv4 = 1;
 149                         if (item->mask != NULL &&
 150                             ((const struct rte_flow_item_ipv4 *)
 151                             item->mask)->hdr.next_proto_id)
 152                                 next_protocol =
 153                                     ((const struct rte_flow_item_ipv4 *)
 154                                       (item->spec))->hdr.next_proto_id &
 155                                     ((const struct rte_flow_item_ipv4 *)
 156                                       (item->mask))->hdr.next_proto_id;
 157                         if ((next_protocol == IPPROTO_IPIP ||
 158                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 159                                 attr->attr = 0;
 160                         break;
 161                 case RTE_FLOW_ITEM_TYPE_IPV6:
 162                         if (!attr->ipv4)
 163                                 attr->ipv6 = 1;
 164                         if (item->mask != NULL &&
 165                             ((const struct rte_flow_item_ipv6 *)
 166                             item->mask)->hdr.proto)
 167                                 next_protocol =
 168                                     ((const struct rte_flow_item_ipv6 *)
 169                                       (item->spec))->hdr.proto &
 170                                     ((const struct rte_flow_item_ipv6 *)
 171                                       (item->mask))->hdr.proto;
 172                         if ((next_protocol == IPPROTO_IPIP ||
 173                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 174                                 attr->attr = 0;
 175                         break;
 176                 case RTE_FLOW_ITEM_TYPE_UDP:
 177                         if (!attr->tcp)
 178                                 attr->udp = 1;
 179                         break;
 180                 case RTE_FLOW_ITEM_TYPE_TCP:
 181                         if (!attr->udp)
 182                                 attr->tcp = 1;
 183                         break;
 184                 default:
 185                         break;
 186                 }
 187         }
 188         attr->valid = 1;
 189 }
 190
 191 /*
 192  * Convert rte_mtr_color to mlx5 color.
 193  *
 194  * @param[in] rcol
 195  *   rte_mtr_color.
 196  *
 197  * @return
 198  *   mlx5 color.
 199  */
 200 static inline int
 201 rte_col_2_mlx5_col(enum rte_color rcol)
 202 {
 203         switch (rcol) {
 204         case RTE_COLOR_GREEN:
 205                 return MLX5_FLOW_COLOR_GREEN;
 206         case RTE_COLOR_YELLOW:
 207                 return MLX5_FLOW_COLOR_YELLOW;
 208         case RTE_COLOR_RED:
 209                 return MLX5_FLOW_COLOR_RED;
 210         default:
 211                 break;
 212         }
 213         return MLX5_FLOW_COLOR_UNDEFINED;
 214 }
 215
 216 struct field_modify_info {
 217         uint32_t size; /* Size of field in protocol header, in bytes. */
 218         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 219         enum mlx5_modification_field id;
 220 };
 221
 222 struct field_modify_info modify_eth[] = {
 223         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 224         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 225         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 226         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 227         {0, 0, 0},
 228 };
 229
 230 struct field_modify_info modify_vlan_out_first_vid[] = {
 231         /* Size in bits !!! */
 232         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv4[] = {
 237         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 239         {4, 12, MLX5_MODI_OUT_SIPV4},
 240         {4, 16, MLX5_MODI_OUT_DIPV4},
 241         {0, 0, 0},
 242 };
 243
 244 struct field_modify_info modify_ipv6[] = {
 245         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 246         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 247         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 248         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 249         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 250         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 251         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 252         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 253         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 254         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 255         {0, 0, 0},
 256 };
 257
 258 struct field_modify_info modify_udp[] = {
 259         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 260         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 261         {0, 0, 0},
 262 };
 263
 264 struct field_modify_info modify_tcp[] = {
 265         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 266         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 267         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 268         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 269         {0, 0, 0},
 270 };
 271
 272 static const struct rte_flow_item *
 273 mlx5_flow_find_tunnel_item(const struct rte_flow_item *item)
 274 {
 275         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 276                 switch (item->type) {
 277                 default:
 278                         break;
 279                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 280                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 281                 case RTE_FLOW_ITEM_TYPE_GRE:
 282                 case RTE_FLOW_ITEM_TYPE_MPLS:
 283                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 284                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 285                         return item;
 286                 case RTE_FLOW_ITEM_TYPE_IPV4:
 287                 case RTE_FLOW_ITEM_TYPE_IPV6:
 288                         if (item[1].type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 289                             item[1].type == RTE_FLOW_ITEM_TYPE_IPV6)
 290                                 return item;
 291                         break;
 292                 }
 293         }
 294         return NULL;
 295 }
 296
 297 static void
 298 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 299                           uint8_t next_protocol, uint64_t *item_flags,
 300                           int *tunnel)
 301 {
 302         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 303                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 304         if (next_protocol == IPPROTO_IPIP) {
 305                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 306                 *tunnel = 1;
 307         }
 308         if (next_protocol == IPPROTO_IPV6) {
 309                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 310                 *tunnel = 1;
 311         }
 312 }
 313
 314 static inline struct mlx5_hlist *
 315 flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
 316                      const char *name, uint32_t size, bool direct_key,
 317                      bool lcores_share, void *ctx,
 318                      mlx5_list_create_cb cb_create,
 319                      mlx5_list_match_cb cb_match,
 320                      mlx5_list_remove_cb cb_remove,
 321                      mlx5_list_clone_cb cb_clone,
 322                      mlx5_list_clone_free_cb cb_clone_free)
 323 {
 324         struct mlx5_hlist *hl;
 325         struct mlx5_hlist *expected = NULL;
 326         char s[MLX5_NAME_SIZE];
 327
 328         hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 329         if (likely(hl))
 330                 return hl;
 331         snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
 332         hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
 333                         ctx, cb_create, cb_match, cb_remove, cb_clone,
 334                         cb_clone_free);
 335         if (!hl) {
 336                 DRV_LOG(ERR, "%s hash creation failed", name);
 337                 rte_errno = ENOMEM;
 338                 return NULL;
 339         }
 340         if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
 341                                          __ATOMIC_SEQ_CST,
 342                                          __ATOMIC_SEQ_CST)) {
 343                 mlx5_hlist_destroy(hl);
 344                 hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 345         }
 346         return hl;
 347 }
 348
 349 /* Update VLAN's VID/PCP based on input rte_flow_action.
 350  *
 351  * @param[in] action
 352  *   Pointer to struct rte_flow_action.
 353  * @param[out] vlan
 354  *   Pointer to struct rte_vlan_hdr.
 355  */
 356 static void
 357 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 358                          struct rte_vlan_hdr *vlan)
 359 {
 360         uint16_t vlan_tci;
 361         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 362                 vlan_tci =
 363                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 364                                                action->conf)->vlan_pcp;
 365                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 366                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 367                 vlan->vlan_tci |= vlan_tci;
 368         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 369                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 370                 vlan->vlan_tci |= rte_be_to_cpu_16
 371                     (((const struct rte_flow_action_of_set_vlan_vid *)
 372                                              action->conf)->vlan_vid);
 373         }
 374 }
 375
 376 /**
 377  * Fetch 1, 2, 3 or 4 byte field from the byte array
 378  * and return as unsigned integer in host-endian format.
 379  *
 380  * @param[in] data
 381  *   Pointer to data array.
 382  * @param[in] size
 383  *   Size of field to extract.
 384  *
 385  * @return
 386  *   converted field in host endian format.
 387  */
 388 static inline uint32_t
 389 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 390 {
 391         uint32_t ret;
 392
 393         switch (size) {
 394         case 1:
 395                 ret = *data;
 396                 break;
 397         case 2:
 398                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 399                 break;
 400         case 3:
 401                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 402                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 403                 break;
 404         case 4:
 405                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 406                 break;
 407         default:
 408                 MLX5_ASSERT(false);
 409                 ret = 0;
 410                 break;
 411         }
 412         return ret;
 413 }
 414
 415 /**
 416  * Convert modify-header action to DV specification.
 417  *
 418  * Data length of each action is determined by provided field description
 419  * and the item mask. Data bit offset and width of each action is determined
 420  * by provided item mask.
 421  *
 422  * @param[in] item
 423  *   Pointer to item specification.
 424  * @param[in] field
 425  *   Pointer to field modification information.
 426  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 427  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 428  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 429  * @param[in] dcopy
 430  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 431  *   Negative offset value sets the same offset as source offset.
 432  *   size field is ignored, value is taken from source field.
 433  * @param[in,out] resource
 434  *   Pointer to the modify-header resource.
 435  * @param[in] type
 436  *   Type of modification.
 437  * @param[out] error
 438  *   Pointer to the error structure.
 439  *
 440  * @return
 441  *   0 on success, a negative errno value otherwise and rte_errno is set.
 442  */
 443 static int
 444 flow_dv_convert_modify_action(struct rte_flow_item *item,
 445                               struct field_modify_info *field,
 446                               struct field_modify_info *dcopy,
 447                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 448                               uint32_t type, struct rte_flow_error *error)
 449 {
 450         uint32_t i = resource->actions_num;
 451         struct mlx5_modification_cmd *actions = resource->actions;
 452         uint32_t carry_b = 0;
 453
 454         /*
 455          * The item and mask are provided in big-endian format.
 456          * The fields should be presented as in big-endian format either.
 457          * Mask must be always present, it defines the actual field width.
 458          */
 459         MLX5_ASSERT(item->mask);
 460         MLX5_ASSERT(field->size);
 461         do {
 462                 uint32_t size_b;
 463                 uint32_t off_b;
 464                 uint32_t mask;
 465                 uint32_t data;
 466                 bool next_field = true;
 467                 bool next_dcopy = true;
 468
 469                 if (i >= MLX5_MAX_MODIFY_NUM)
 470                         return rte_flow_error_set(error, EINVAL,
 471                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 472                                  "too many items to modify");
 473                 /* Fetch variable byte size mask from the array. */
 474                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 475                                            field->offset, field->size);
 476                 if (!mask) {
 477                         ++field;
 478                         continue;
 479                 }
 480                 /* Deduce actual data width in bits from mask value. */
 481                 off_b = rte_bsf32(mask) + carry_b;
 482                 size_b = sizeof(uint32_t) * CHAR_BIT -
 483                          off_b - __builtin_clz(mask);
 484                 MLX5_ASSERT(size_b);
 485                 actions[i] = (struct mlx5_modification_cmd) {
 486                         .action_type = type,
 487                         .field = field->id,
 488                         .offset = off_b,
 489                         .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ?
 490                                 0 : size_b,
 491                 };
 492                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 493                         MLX5_ASSERT(dcopy);
 494                         actions[i].dst_field = dcopy->id;
 495                         actions[i].dst_offset =
 496                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 497                         /* Convert entire record to big-endian format. */
 498                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 499                         /*
 500                          * Destination field overflow. Copy leftovers of
 501                          * a source field to the next destination field.
 502                          */
 503                         carry_b = 0;
 504                         if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) &&
 505                             dcopy->size != 0) {
 506                                 actions[i].length =
 507                                         dcopy->size * CHAR_BIT - dcopy->offset;
 508                                 carry_b = actions[i].length;
 509                                 next_field = false;
 510                         }
 511                         /*
 512                          * Not enough bits in a source filed to fill a
 513                          * destination field. Switch to the next source.
 514                          */
 515                         if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) &&
 516                             (size_b == field->size * CHAR_BIT - off_b)) {
 517                                 actions[i].length =
 518                                         field->size * CHAR_BIT - off_b;
 519                                 dcopy->offset += actions[i].length;
 520                                 next_dcopy = false;
 521                         }
 522                         if (next_dcopy)
 523                                 ++dcopy;
 524                 } else {
 525                         MLX5_ASSERT(item->spec);
 526                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 527                                                    field->offset, field->size);
 528                         /* Shift out the trailing masked bits from data. */
 529                         data = (data & mask) >> off_b;
 530                         actions[i].data1 = rte_cpu_to_be_32(data);
 531                 }
 532                 /* Convert entire record to expected big-endian format. */
 533                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 534                 if (next_field)
 535                         ++field;
 536                 ++i;
 537         } while (field->size);
 538         if (resource->actions_num == i)
 539                 return rte_flow_error_set(error, EINVAL,
 540                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 541                                           "invalid modification flow item");
 542         resource->actions_num = i;
 543         return 0;
 544 }
 545
 546 /**
 547  * Convert modify-header set IPv4 address action to DV specification.
 548  *
 549  * @param[in,out] resource
 550  *   Pointer to the modify-header resource.
 551  * @param[in] action
 552  *   Pointer to action specification.
 553  * @param[out] error
 554  *   Pointer to the error structure.
 555  *
 556  * @return
 557  *   0 on success, a negative errno value otherwise and rte_errno is set.
 558  */
 559 static int
 560 flow_dv_convert_action_modify_ipv4
 561                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 562                          const struct rte_flow_action *action,
 563                          struct rte_flow_error *error)
 564 {
 565         const struct rte_flow_action_set_ipv4 *conf =
 566                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 567         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 568         struct rte_flow_item_ipv4 ipv4;
 569         struct rte_flow_item_ipv4 ipv4_mask;
 570
 571         memset(&ipv4, 0, sizeof(ipv4));
 572         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 573         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 574                 ipv4.hdr.src_addr = conf->ipv4_addr;
 575                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 576         } else {
 577                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 578                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 579         }
 580         item.spec = &ipv4;
 581         item.mask = &ipv4_mask;
 582         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 583                                              MLX5_MODIFICATION_TYPE_SET, error);
 584 }
 585
 586 /**
 587  * Convert modify-header set IPv6 address action to DV specification.
 588  *
 589  * @param[in,out] resource
 590  *   Pointer to the modify-header resource.
 591  * @param[in] action
 592  *   Pointer to action specification.
 593  * @param[out] error
 594  *   Pointer to the error structure.
 595  *
 596  * @return
 597  *   0 on success, a negative errno value otherwise and rte_errno is set.
 598  */
 599 static int
 600 flow_dv_convert_action_modify_ipv6
 601                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 602                          const struct rte_flow_action *action,
 603                          struct rte_flow_error *error)
 604 {
 605         const struct rte_flow_action_set_ipv6 *conf =
 606                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 607         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 608         struct rte_flow_item_ipv6 ipv6;
 609         struct rte_flow_item_ipv6 ipv6_mask;
 610
 611         memset(&ipv6, 0, sizeof(ipv6));
 612         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 613         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 614                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 615                        sizeof(ipv6.hdr.src_addr));
 616                 memcpy(&ipv6_mask.hdr.src_addr,
 617                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 618                        sizeof(ipv6.hdr.src_addr));
 619         } else {
 620                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 621                        sizeof(ipv6.hdr.dst_addr));
 622                 memcpy(&ipv6_mask.hdr.dst_addr,
 623                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 624                        sizeof(ipv6.hdr.dst_addr));
 625         }
 626         item.spec = &ipv6;
 627         item.mask = &ipv6_mask;
 628         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 629                                              MLX5_MODIFICATION_TYPE_SET, error);
 630 }
 631
 632 /**
 633  * Convert modify-header set MAC address action to DV specification.
 634  *
 635  * @param[in,out] resource
 636  *   Pointer to the modify-header resource.
 637  * @param[in] action
 638  *   Pointer to action specification.
 639  * @param[out] error
 640  *   Pointer to the error structure.
 641  *
 642  * @return
 643  *   0 on success, a negative errno value otherwise and rte_errno is set.
 644  */
 645 static int
 646 flow_dv_convert_action_modify_mac
 647                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 648                          const struct rte_flow_action *action,
 649                          struct rte_flow_error *error)
 650 {
 651         const struct rte_flow_action_set_mac *conf =
 652                 (const struct rte_flow_action_set_mac *)(action->conf);
 653         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 654         struct rte_flow_item_eth eth;
 655         struct rte_flow_item_eth eth_mask;
 656
 657         memset(&eth, 0, sizeof(eth));
 658         memset(&eth_mask, 0, sizeof(eth_mask));
 659         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 660                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 661                        sizeof(eth.src.addr_bytes));
 662                 memcpy(&eth_mask.src.addr_bytes,
 663                        &rte_flow_item_eth_mask.src.addr_bytes,
 664                        sizeof(eth_mask.src.addr_bytes));
 665         } else {
 666                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 667                        sizeof(eth.dst.addr_bytes));
 668                 memcpy(&eth_mask.dst.addr_bytes,
 669                        &rte_flow_item_eth_mask.dst.addr_bytes,
 670                        sizeof(eth_mask.dst.addr_bytes));
 671         }
 672         item.spec = &eth;
 673         item.mask = &eth_mask;
 674         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 675                                              MLX5_MODIFICATION_TYPE_SET, error);
 676 }
 677
 678 /**
 679  * Convert modify-header set VLAN VID action to DV specification.
 680  *
 681  * @param[in,out] resource
 682  *   Pointer to the modify-header resource.
 683  * @param[in] action
 684  *   Pointer to action specification.
 685  * @param[out] error
 686  *   Pointer to the error structure.
 687  *
 688  * @return
 689  *   0 on success, a negative errno value otherwise and rte_errno is set.
 690  */
 691 static int
 692 flow_dv_convert_action_modify_vlan_vid
 693                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 694                          const struct rte_flow_action *action,
 695                          struct rte_flow_error *error)
 696 {
 697         const struct rte_flow_action_of_set_vlan_vid *conf =
 698                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 699         int i = resource->actions_num;
 700         struct mlx5_modification_cmd *actions = resource->actions;
 701         struct field_modify_info *field = modify_vlan_out_first_vid;
 702
 703         if (i >= MLX5_MAX_MODIFY_NUM)
 704                 return rte_flow_error_set(error, EINVAL,
 705                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 706                          "too many items to modify");
 707         actions[i] = (struct mlx5_modification_cmd) {
 708                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 709                 .field = field->id,
 710                 .length = field->size,
 711                 .offset = field->offset,
 712         };
 713         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 714         actions[i].data1 = conf->vlan_vid;
 715         actions[i].data1 = actions[i].data1 << 16;
 716         resource->actions_num = ++i;
 717         return 0;
 718 }
 719
 720 /**
 721  * Convert modify-header set TP action to DV specification.
 722  *
 723  * @param[in,out] resource
 724  *   Pointer to the modify-header resource.
 725  * @param[in] action
 726  *   Pointer to action specification.
 727  * @param[in] items
 728  *   Pointer to rte_flow_item objects list.
 729  * @param[in] attr
 730  *   Pointer to flow attributes structure.
 731  * @param[in] dev_flow
 732  *   Pointer to the sub flow.
 733  * @param[in] tunnel_decap
 734  *   Whether action is after tunnel decapsulation.
 735  * @param[out] error
 736  *   Pointer to the error structure.
 737  *
 738  * @return
 739  *   0 on success, a negative errno value otherwise and rte_errno is set.
 740  */
 741 static int
 742 flow_dv_convert_action_modify_tp
 743                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 744                          const struct rte_flow_action *action,
 745                          const struct rte_flow_item *items,
 746                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 747                          bool tunnel_decap, struct rte_flow_error *error)
 748 {
 749         const struct rte_flow_action_set_tp *conf =
 750                 (const struct rte_flow_action_set_tp *)(action->conf);
 751         struct rte_flow_item item;
 752         struct rte_flow_item_udp udp;
 753         struct rte_flow_item_udp udp_mask;
 754         struct rte_flow_item_tcp tcp;
 755         struct rte_flow_item_tcp tcp_mask;
 756         struct field_modify_info *field;
 757
 758         if (!attr->valid)
 759                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 760         if (attr->udp) {
 761                 memset(&udp, 0, sizeof(udp));
 762                 memset(&udp_mask, 0, sizeof(udp_mask));
 763                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 764                         udp.hdr.src_port = conf->port;
 765                         udp_mask.hdr.src_port =
 766                                         rte_flow_item_udp_mask.hdr.src_port;
 767                 } else {
 768                         udp.hdr.dst_port = conf->port;
 769                         udp_mask.hdr.dst_port =
 770                                         rte_flow_item_udp_mask.hdr.dst_port;
 771                 }
 772                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 773                 item.spec = &udp;
 774                 item.mask = &udp_mask;
 775                 field = modify_udp;
 776         } else {
 777                 MLX5_ASSERT(attr->tcp);
 778                 memset(&tcp, 0, sizeof(tcp));
 779                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 780                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 781                         tcp.hdr.src_port = conf->port;
 782                         tcp_mask.hdr.src_port =
 783                                         rte_flow_item_tcp_mask.hdr.src_port;
 784                 } else {
 785                         tcp.hdr.dst_port = conf->port;
 786                         tcp_mask.hdr.dst_port =
 787                                         rte_flow_item_tcp_mask.hdr.dst_port;
 788                 }
 789                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 790                 item.spec = &tcp;
 791                 item.mask = &tcp_mask;
 792                 field = modify_tcp;
 793         }
 794         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 795                                              MLX5_MODIFICATION_TYPE_SET, error);
 796 }
 797
 798 /**
 799  * Convert modify-header set TTL action to DV specification.
 800  *
 801  * @param[in,out] resource
 802  *   Pointer to the modify-header resource.
 803  * @param[in] action
 804  *   Pointer to action specification.
 805  * @param[in] items
 806  *   Pointer to rte_flow_item objects list.
 807  * @param[in] attr
 808  *   Pointer to flow attributes structure.
 809  * @param[in] dev_flow
 810  *   Pointer to the sub flow.
 811  * @param[in] tunnel_decap
 812  *   Whether action is after tunnel decapsulation.
 813  * @param[out] error
 814  *   Pointer to the error structure.
 815  *
 816  * @return
 817  *   0 on success, a negative errno value otherwise and rte_errno is set.
 818  */
 819 static int
 820 flow_dv_convert_action_modify_ttl
 821                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 822                          const struct rte_flow_action *action,
 823                          const struct rte_flow_item *items,
 824                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 825                          bool tunnel_decap, struct rte_flow_error *error)
 826 {
 827         const struct rte_flow_action_set_ttl *conf =
 828                 (const struct rte_flow_action_set_ttl *)(action->conf);
 829         struct rte_flow_item item;
 830         struct rte_flow_item_ipv4 ipv4;
 831         struct rte_flow_item_ipv4 ipv4_mask;
 832         struct rte_flow_item_ipv6 ipv6;
 833         struct rte_flow_item_ipv6 ipv6_mask;
 834         struct field_modify_info *field;
 835
 836         if (!attr->valid)
 837                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 838         if (attr->ipv4) {
 839                 memset(&ipv4, 0, sizeof(ipv4));
 840                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 841                 ipv4.hdr.time_to_live = conf->ttl_value;
 842                 ipv4_mask.hdr.time_to_live = 0xFF;
 843                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 844                 item.spec = &ipv4;
 845                 item.mask = &ipv4_mask;
 846                 field = modify_ipv4;
 847         } else {
 848                 MLX5_ASSERT(attr->ipv6);
 849                 memset(&ipv6, 0, sizeof(ipv6));
 850                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 851                 ipv6.hdr.hop_limits = conf->ttl_value;
 852                 ipv6_mask.hdr.hop_limits = 0xFF;
 853                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 854                 item.spec = &ipv6;
 855                 item.mask = &ipv6_mask;
 856                 field = modify_ipv6;
 857         }
 858         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 859                                              MLX5_MODIFICATION_TYPE_SET, error);
 860 }
 861
 862 /**
 863  * Convert modify-header decrement TTL action to DV specification.
 864  *
 865  * @param[in,out] resource
 866  *   Pointer to the modify-header resource.
 867  * @param[in] action
 868  *   Pointer to action specification.
 869  * @param[in] items
 870  *   Pointer to rte_flow_item objects list.
 871  * @param[in] attr
 872  *   Pointer to flow attributes structure.
 873  * @param[in] dev_flow
 874  *   Pointer to the sub flow.
 875  * @param[in] tunnel_decap
 876  *   Whether action is after tunnel decapsulation.
 877  * @param[out] error
 878  *   Pointer to the error structure.
 879  *
 880  * @return
 881  *   0 on success, a negative errno value otherwise and rte_errno is set.
 882  */
 883 static int
 884 flow_dv_convert_action_modify_dec_ttl
 885                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 886                          const struct rte_flow_item *items,
 887                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 888                          bool tunnel_decap, struct rte_flow_error *error)
 889 {
 890         struct rte_flow_item item;
 891         struct rte_flow_item_ipv4 ipv4;
 892         struct rte_flow_item_ipv4 ipv4_mask;
 893         struct rte_flow_item_ipv6 ipv6;
 894         struct rte_flow_item_ipv6 ipv6_mask;
 895         struct field_modify_info *field;
 896
 897         if (!attr->valid)
 898                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 899         if (attr->ipv4) {
 900                 memset(&ipv4, 0, sizeof(ipv4));
 901                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 902                 ipv4.hdr.time_to_live = 0xFF;
 903                 ipv4_mask.hdr.time_to_live = 0xFF;
 904                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 905                 item.spec = &ipv4;
 906                 item.mask = &ipv4_mask;
 907                 field = modify_ipv4;
 908         } else {
 909                 MLX5_ASSERT(attr->ipv6);
 910                 memset(&ipv6, 0, sizeof(ipv6));
 911                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 912                 ipv6.hdr.hop_limits = 0xFF;
 913                 ipv6_mask.hdr.hop_limits = 0xFF;
 914                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 915                 item.spec = &ipv6;
 916                 item.mask = &ipv6_mask;
 917                 field = modify_ipv6;
 918         }
 919         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 920                                              MLX5_MODIFICATION_TYPE_ADD, error);
 921 }
 922
 923 /**
 924  * Convert modify-header increment/decrement TCP Sequence number
 925  * to DV specification.
 926  *
 927  * @param[in,out] resource
 928  *   Pointer to the modify-header resource.
 929  * @param[in] action
 930  *   Pointer to action specification.
 931  * @param[out] error
 932  *   Pointer to the error structure.
 933  *
 934  * @return
 935  *   0 on success, a negative errno value otherwise and rte_errno is set.
 936  */
 937 static int
 938 flow_dv_convert_action_modify_tcp_seq
 939                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 940                          const struct rte_flow_action *action,
 941                          struct rte_flow_error *error)
 942 {
 943         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 944         uint64_t value = rte_be_to_cpu_32(*conf);
 945         struct rte_flow_item item;
 946         struct rte_flow_item_tcp tcp;
 947         struct rte_flow_item_tcp tcp_mask;
 948
 949         memset(&tcp, 0, sizeof(tcp));
 950         memset(&tcp_mask, 0, sizeof(tcp_mask));
 951         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 952                 /*
 953                  * The HW has no decrement operation, only increment operation.
 954                  * To simulate decrement X from Y using increment operation
 955                  * we need to add UINT32_MAX X times to Y.
 956                  * Each adding of UINT32_MAX decrements Y by 1.
 957                  */
 958                 value *= UINT32_MAX;
 959         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 960         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 961         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 962         item.spec = &tcp;
 963         item.mask = &tcp_mask;
 964         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 965                                              MLX5_MODIFICATION_TYPE_ADD, error);
 966 }
 967
 968 /**
 969  * Convert modify-header increment/decrement TCP Acknowledgment number
 970  * to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_modify_tcp_ack
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 989         uint64_t value = rte_be_to_cpu_32(*conf);
 990         struct rte_flow_item item;
 991         struct rte_flow_item_tcp tcp;
 992         struct rte_flow_item_tcp tcp_mask;
 993
 994         memset(&tcp, 0, sizeof(tcp));
 995         memset(&tcp_mask, 0, sizeof(tcp_mask));
 996         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 997                 /*
 998                  * The HW has no decrement operation, only increment operation.
 999                  * To simulate decrement X from Y using increment operation
1000                  * we need to add UINT32_MAX X times to Y.
1001                  * Each adding of UINT32_MAX decrements Y by 1.
1002                  */
1003                 value *= UINT32_MAX;
1004         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
1005         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
1006         item.type = RTE_FLOW_ITEM_TYPE_TCP;
1007         item.spec = &tcp;
1008         item.mask = &tcp_mask;
1009         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
1010                                              MLX5_MODIFICATION_TYPE_ADD, error);
1011 }
1012
1013 static enum mlx5_modification_field reg_to_field[] = {
1014         [REG_NON] = MLX5_MODI_OUT_NONE,
1015         [REG_A] = MLX5_MODI_META_DATA_REG_A,
1016         [REG_B] = MLX5_MODI_META_DATA_REG_B,
1017         [REG_C_0] = MLX5_MODI_META_REG_C_0,
1018         [REG_C_1] = MLX5_MODI_META_REG_C_1,
1019         [REG_C_2] = MLX5_MODI_META_REG_C_2,
1020         [REG_C_3] = MLX5_MODI_META_REG_C_3,
1021         [REG_C_4] = MLX5_MODI_META_REG_C_4,
1022         [REG_C_5] = MLX5_MODI_META_REG_C_5,
1023         [REG_C_6] = MLX5_MODI_META_REG_C_6,
1024         [REG_C_7] = MLX5_MODI_META_REG_C_7,
1025 };
1026
1027 /**
1028  * Convert register set to DV specification.
1029  *
1030  * @param[in,out] resource
1031  *   Pointer to the modify-header resource.
1032  * @param[in] action
1033  *   Pointer to action specification.
1034  * @param[out] error
1035  *   Pointer to the error structure.
1036  *
1037  * @return
1038  *   0 on success, a negative errno value otherwise and rte_errno is set.
1039  */
1040 static int
1041 flow_dv_convert_action_set_reg
1042                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1043                          const struct rte_flow_action *action,
1044                          struct rte_flow_error *error)
1045 {
1046         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
1047         struct mlx5_modification_cmd *actions = resource->actions;
1048         uint32_t i = resource->actions_num;
1049
1050         if (i >= MLX5_MAX_MODIFY_NUM)
1051                 return rte_flow_error_set(error, EINVAL,
1052                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1053                                           "too many items to modify");
1054         MLX5_ASSERT(conf->id != REG_NON);
1055         MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
1056         actions[i] = (struct mlx5_modification_cmd) {
1057                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1058                 .field = reg_to_field[conf->id],
1059                 .offset = conf->offset,
1060                 .length = conf->length,
1061         };
1062         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1063         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1064         ++i;
1065         resource->actions_num = i;
1066         return 0;
1067 }
1068
1069 /**
1070  * Convert SET_TAG action to DV specification.
1071  *
1072  * @param[in] dev
1073  *   Pointer to the rte_eth_dev structure.
1074  * @param[in,out] resource
1075  *   Pointer to the modify-header resource.
1076  * @param[in] conf
1077  *   Pointer to action specification.
1078  * @param[out] error
1079  *   Pointer to the error structure.
1080  *
1081  * @return
1082  *   0 on success, a negative errno value otherwise and rte_errno is set.
1083  */
1084 static int
1085 flow_dv_convert_action_set_tag
1086                         (struct rte_eth_dev *dev,
1087                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1088                          const struct rte_flow_action_set_tag *conf,
1089                          struct rte_flow_error *error)
1090 {
1091         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1092         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1093         struct rte_flow_item item = {
1094                 .spec = &data,
1095                 .mask = &mask,
1096         };
1097         struct field_modify_info reg_c_x[] = {
1098                 [1] = {0, 0, 0},
1099         };
1100         enum mlx5_modification_field reg_type;
1101         int ret;
1102
1103         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1104         if (ret < 0)
1105                 return ret;
1106         MLX5_ASSERT(ret != REG_NON);
1107         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1108         reg_type = reg_to_field[ret];
1109         MLX5_ASSERT(reg_type > 0);
1110         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1111         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1112                                              MLX5_MODIFICATION_TYPE_SET, error);
1113 }
1114
1115 /**
1116  * Convert internal COPY_REG action to DV specification.
1117  *
1118  * @param[in] dev
1119  *   Pointer to the rte_eth_dev structure.
1120  * @param[in,out] res
1121  *   Pointer to the modify-header resource.
1122  * @param[in] action
1123  *   Pointer to action specification.
1124  * @param[out] error
1125  *   Pointer to the error structure.
1126  *
1127  * @return
1128  *   0 on success, a negative errno value otherwise and rte_errno is set.
1129  */
1130 static int
1131 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1132                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1133                                  const struct rte_flow_action *action,
1134                                  struct rte_flow_error *error)
1135 {
1136         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1137         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1138         struct rte_flow_item item = {
1139                 .spec = NULL,
1140                 .mask = &mask,
1141         };
1142         struct field_modify_info reg_src[] = {
1143                 {4, 0, reg_to_field[conf->src]},
1144                 {0, 0, 0},
1145         };
1146         struct field_modify_info reg_dst = {
1147                 .offset = 0,
1148                 .id = reg_to_field[conf->dst],
1149         };
1150         /* Adjust reg_c[0] usage according to reported mask. */
1151         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1152                 struct mlx5_priv *priv = dev->data->dev_private;
1153                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1154
1155                 MLX5_ASSERT(reg_c0);
1156                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1157                 if (conf->dst == REG_C_0) {
1158                         /* Copy to reg_c[0], within mask only. */
1159                         reg_dst.offset = rte_bsf32(reg_c0);
1160                         mask = rte_cpu_to_be_32(reg_c0 >> reg_dst.offset);
1161                 } else {
1162                         reg_dst.offset = 0;
1163                         mask = rte_cpu_to_be_32(reg_c0);
1164                 }
1165         }
1166         return flow_dv_convert_modify_action(&item,
1167                                              reg_src, &reg_dst, res,
1168                                              MLX5_MODIFICATION_TYPE_COPY,
1169                                              error);
1170 }
1171
1172 /**
1173  * Convert MARK action to DV specification. This routine is used
1174  * in extensive metadata only and requires metadata register to be
1175  * handled. In legacy mode hardware tag resource is engaged.
1176  *
1177  * @param[in] dev
1178  *   Pointer to the rte_eth_dev structure.
1179  * @param[in] conf
1180  *   Pointer to MARK action specification.
1181  * @param[in,out] resource
1182  *   Pointer to the modify-header resource.
1183  * @param[out] error
1184  *   Pointer to the error structure.
1185  *
1186  * @return
1187  *   0 on success, a negative errno value otherwise and rte_errno is set.
1188  */
1189 static int
1190 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1191                             const struct rte_flow_action_mark *conf,
1192                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1193                             struct rte_flow_error *error)
1194 {
1195         struct mlx5_priv *priv = dev->data->dev_private;
1196         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1197                                            priv->sh->dv_mark_mask);
1198         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1199         struct rte_flow_item item = {
1200                 .spec = &data,
1201                 .mask = &mask,
1202         };
1203         struct field_modify_info reg_c_x[] = {
1204                 [1] = {0, 0, 0},
1205         };
1206         int reg;
1207
1208         if (!mask)
1209                 return rte_flow_error_set(error, EINVAL,
1210                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1211                                           NULL, "zero mark action mask");
1212         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1213         if (reg < 0)
1214                 return reg;
1215         MLX5_ASSERT(reg > 0);
1216         if (reg == REG_C_0) {
1217                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1218                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1219
1220                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1221                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1222                 mask = rte_cpu_to_be_32(mask << shl_c0);
1223         }
1224         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1225         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1226                                              MLX5_MODIFICATION_TYPE_SET, error);
1227 }
1228
1229 /**
1230  * Get metadata register index for specified steering domain.
1231  *
1232  * @param[in] dev
1233  *   Pointer to the rte_eth_dev structure.
1234  * @param[in] attr
1235  *   Attributes of flow to determine steering domain.
1236  * @param[out] error
1237  *   Pointer to the error structure.
1238  *
1239  * @return
1240  *   positive index on success, a negative errno value otherwise
1241  *   and rte_errno is set.
1242  */
1243 static enum modify_reg
1244 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1245                          const struct rte_flow_attr *attr,
1246                          struct rte_flow_error *error)
1247 {
1248         int reg =
1249                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1250                                           MLX5_METADATA_FDB :
1251                                             attr->egress ?
1252                                             MLX5_METADATA_TX :
1253                                             MLX5_METADATA_RX, 0, error);
1254         if (reg < 0)
1255                 return rte_flow_error_set(error,
1256                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1257                                           NULL, "unavailable "
1258                                           "metadata register");
1259         return reg;
1260 }
1261
1262 /**
1263  * Convert SET_META action to DV specification.
1264  *
1265  * @param[in] dev
1266  *   Pointer to the rte_eth_dev structure.
1267  * @param[in,out] resource
1268  *   Pointer to the modify-header resource.
1269  * @param[in] attr
1270  *   Attributes of flow that includes this item.
1271  * @param[in] conf
1272  *   Pointer to action specification.
1273  * @param[out] error
1274  *   Pointer to the error structure.
1275  *
1276  * @return
1277  *   0 on success, a negative errno value otherwise and rte_errno is set.
1278  */
1279 static int
1280 flow_dv_convert_action_set_meta
1281                         (struct rte_eth_dev *dev,
1282                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1283                          const struct rte_flow_attr *attr,
1284                          const struct rte_flow_action_set_meta *conf,
1285                          struct rte_flow_error *error)
1286 {
1287         uint32_t mask = rte_cpu_to_be_32(conf->mask);
1288         uint32_t data = rte_cpu_to_be_32(conf->data) & mask;
1289         struct rte_flow_item item = {
1290                 .spec = &data,
1291                 .mask = &mask,
1292         };
1293         struct field_modify_info reg_c_x[] = {
1294                 [1] = {0, 0, 0},
1295         };
1296         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1297
1298         if (reg < 0)
1299                 return reg;
1300         MLX5_ASSERT(reg != REG_NON);
1301         if (reg == REG_C_0) {
1302                 struct mlx5_priv *priv = dev->data->dev_private;
1303                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1304                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1305
1306                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1307                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1308                 mask = rte_cpu_to_be_32(mask << shl_c0);
1309         }
1310         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1311         /* The routine expects parameters in memory as big-endian ones. */
1312         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1313                                              MLX5_MODIFICATION_TYPE_SET, error);
1314 }
1315
1316 /**
1317  * Convert modify-header set IPv4 DSCP action to DV specification.
1318  *
1319  * @param[in,out] resource
1320  *   Pointer to the modify-header resource.
1321  * @param[in] action
1322  *   Pointer to action specification.
1323  * @param[out] error
1324  *   Pointer to the error structure.
1325  *
1326  * @return
1327  *   0 on success, a negative errno value otherwise and rte_errno is set.
1328  */
1329 static int
1330 flow_dv_convert_action_modify_ipv4_dscp
1331                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1332                          const struct rte_flow_action *action,
1333                          struct rte_flow_error *error)
1334 {
1335         const struct rte_flow_action_set_dscp *conf =
1336                 (const struct rte_flow_action_set_dscp *)(action->conf);
1337         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1338         struct rte_flow_item_ipv4 ipv4;
1339         struct rte_flow_item_ipv4 ipv4_mask;
1340
1341         memset(&ipv4, 0, sizeof(ipv4));
1342         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1343         ipv4.hdr.type_of_service = conf->dscp;
1344         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1345         item.spec = &ipv4;
1346         item.mask = &ipv4_mask;
1347         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1348                                              MLX5_MODIFICATION_TYPE_SET, error);
1349 }
1350
1351 /**
1352  * Convert modify-header set IPv6 DSCP action to DV specification.
1353  *
1354  * @param[in,out] resource
1355  *   Pointer to the modify-header resource.
1356  * @param[in] action
1357  *   Pointer to action specification.
1358  * @param[out] error
1359  *   Pointer to the error structure.
1360  *
1361  * @return
1362  *   0 on success, a negative errno value otherwise and rte_errno is set.
1363  */
1364 static int
1365 flow_dv_convert_action_modify_ipv6_dscp
1366                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1367                          const struct rte_flow_action *action,
1368                          struct rte_flow_error *error)
1369 {
1370         const struct rte_flow_action_set_dscp *conf =
1371                 (const struct rte_flow_action_set_dscp *)(action->conf);
1372         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1373         struct rte_flow_item_ipv6 ipv6;
1374         struct rte_flow_item_ipv6 ipv6_mask;
1375
1376         memset(&ipv6, 0, sizeof(ipv6));
1377         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1378         /*
1379          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1380          * rdma-core only accept the DSCP bits byte aligned start from
1381          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1382          * bits in IPv6 case as rdma-core requires byte aligned value.
1383          */
1384         ipv6.hdr.vtc_flow = conf->dscp;
1385         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1386         item.spec = &ipv6;
1387         item.mask = &ipv6_mask;
1388         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1389                                              MLX5_MODIFICATION_TYPE_SET, error);
1390 }
1391
1392 static int
1393 mlx5_flow_item_field_width(struct mlx5_priv *priv,
1394                            enum rte_flow_field_id field)
1395 {
1396         switch (field) {
1397         case RTE_FLOW_FIELD_START:
1398                 return 32;
1399         case RTE_FLOW_FIELD_MAC_DST:
1400         case RTE_FLOW_FIELD_MAC_SRC:
1401                 return 48;
1402         case RTE_FLOW_FIELD_VLAN_TYPE:
1403                 return 16;
1404         case RTE_FLOW_FIELD_VLAN_ID:
1405                 return 12;
1406         case RTE_FLOW_FIELD_MAC_TYPE:
1407                 return 16;
1408         case RTE_FLOW_FIELD_IPV4_DSCP:
1409                 return 6;
1410         case RTE_FLOW_FIELD_IPV4_TTL:
1411                 return 8;
1412         case RTE_FLOW_FIELD_IPV4_SRC:
1413         case RTE_FLOW_FIELD_IPV4_DST:
1414                 return 32;
1415         case RTE_FLOW_FIELD_IPV6_DSCP:
1416                 return 6;
1417         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1418                 return 8;
1419         case RTE_FLOW_FIELD_IPV6_SRC:
1420         case RTE_FLOW_FIELD_IPV6_DST:
1421                 return 128;
1422         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1423         case RTE_FLOW_FIELD_TCP_PORT_DST:
1424                 return 16;
1425         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1426         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1427                 return 32;
1428         case RTE_FLOW_FIELD_TCP_FLAGS:
1429                 return 9;
1430         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1431         case RTE_FLOW_FIELD_UDP_PORT_DST:
1432                 return 16;
1433         case RTE_FLOW_FIELD_VXLAN_VNI:
1434         case RTE_FLOW_FIELD_GENEVE_VNI:
1435                 return 24;
1436         case RTE_FLOW_FIELD_GTP_TEID:
1437         case RTE_FLOW_FIELD_TAG:
1438                 return 32;
1439         case RTE_FLOW_FIELD_MARK:
1440                 return __builtin_popcount(priv->sh->dv_mark_mask);
1441         case RTE_FLOW_FIELD_META:
1442                 return __builtin_popcount(priv->sh->dv_meta_mask);
1443         case RTE_FLOW_FIELD_POINTER:
1444         case RTE_FLOW_FIELD_VALUE:
1445                 return 64;
1446         default:
1447                 MLX5_ASSERT(false);
1448         }
1449         return 0;
1450 }
1451
1452 static void
1453 mlx5_flow_field_id_to_modify_info
1454                 (const struct rte_flow_action_modify_data *data,
1455                  struct field_modify_info *info,
1456                  uint32_t *mask, uint32_t *value,
1457                  uint32_t width, uint32_t dst_width,
1458                  uint32_t *shift, struct rte_eth_dev *dev,
1459                  const struct rte_flow_attr *attr,
1460                  struct rte_flow_error *error)
1461 {
1462         struct mlx5_priv *priv = dev->data->dev_private;
1463         uint32_t idx = 0;
1464         uint32_t off = 0;
1465         uint64_t val = 0;
1466         switch (data->field) {
1467         case RTE_FLOW_FIELD_START:
1468                 /* not supported yet */
1469                 MLX5_ASSERT(false);
1470                 break;
1471         case RTE_FLOW_FIELD_MAC_DST:
1472                 off = data->offset > 16 ? data->offset - 16 : 0;
1473                 if (mask) {
1474                         if (data->offset < 16) {
1475                                 info[idx] = (struct field_modify_info){2, 0,
1476                                                 MLX5_MODI_OUT_DMAC_15_0};
1477                                 if (width < 16) {
1478                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1479                                                                  (16 - width));
1480                                         width = 0;
1481                                 } else {
1482                                         mask[idx] = RTE_BE16(0xffff);
1483                                         width -= 16;
1484                                 }
1485                                 if (!width)
1486                                         break;
1487                                 ++idx;
1488                         }
1489                         info[idx] = (struct field_modify_info){4, 4 * idx,
1490                                                 MLX5_MODI_OUT_DMAC_47_16};
1491                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1492                                                       (32 - width)) << off);
1493                 } else {
1494                         if (data->offset < 16)
1495                                 info[idx++] = (struct field_modify_info){2, 0,
1496                                                 MLX5_MODI_OUT_DMAC_15_0};
1497                         info[idx] = (struct field_modify_info){4, off,
1498                                                 MLX5_MODI_OUT_DMAC_47_16};
1499                 }
1500                 break;
1501         case RTE_FLOW_FIELD_MAC_SRC:
1502                 off = data->offset > 16 ? data->offset - 16 : 0;
1503                 if (mask) {
1504                         if (data->offset < 16) {
1505                                 info[idx] = (struct field_modify_info){2, 0,
1506                                                 MLX5_MODI_OUT_SMAC_15_0};
1507                                 if (width < 16) {
1508                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1509                                                                  (16 - width));
1510                                         width = 0;
1511                                 } else {
1512                                         mask[idx] = RTE_BE16(0xffff);
1513                                         width -= 16;
1514                                 }
1515                                 if (!width)
1516                                         break;
1517                                 ++idx;
1518                         }
1519                         info[idx] = (struct field_modify_info){4, 4 * idx,
1520                                                 MLX5_MODI_OUT_SMAC_47_16};
1521                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1522                                                       (32 - width)) << off);
1523                 } else {
1524                         if (data->offset < 16)
1525                                 info[idx++] = (struct field_modify_info){2, 0,
1526                                                 MLX5_MODI_OUT_SMAC_15_0};
1527                         info[idx] = (struct field_modify_info){4, off,
1528                                                 MLX5_MODI_OUT_SMAC_47_16};
1529                 }
1530                 break;
1531         case RTE_FLOW_FIELD_VLAN_TYPE:
1532                 /* not supported yet */
1533                 break;
1534         case RTE_FLOW_FIELD_VLAN_ID:
1535                 info[idx] = (struct field_modify_info){2, 0,
1536                                         MLX5_MODI_OUT_FIRST_VID};
1537                 if (mask)
1538                         mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
1539                 break;
1540         case RTE_FLOW_FIELD_MAC_TYPE:
1541                 info[idx] = (struct field_modify_info){2, 0,
1542                                         MLX5_MODI_OUT_ETHERTYPE};
1543                 if (mask)
1544                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1545                 break;
1546         case RTE_FLOW_FIELD_IPV4_DSCP:
1547                 info[idx] = (struct field_modify_info){1, 0,
1548                                         MLX5_MODI_OUT_IP_DSCP};
1549                 if (mask)
1550                         mask[idx] = 0x3f >> (6 - width);
1551                 break;
1552         case RTE_FLOW_FIELD_IPV4_TTL:
1553                 info[idx] = (struct field_modify_info){1, 0,
1554                                         MLX5_MODI_OUT_IPV4_TTL};
1555                 if (mask)
1556                         mask[idx] = 0xff >> (8 - width);
1557                 break;
1558         case RTE_FLOW_FIELD_IPV4_SRC:
1559                 info[idx] = (struct field_modify_info){4, 0,
1560                                         MLX5_MODI_OUT_SIPV4};
1561                 if (mask)
1562                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1563                                                      (32 - width));
1564                 break;
1565         case RTE_FLOW_FIELD_IPV4_DST:
1566                 info[idx] = (struct field_modify_info){4, 0,
1567                                         MLX5_MODI_OUT_DIPV4};
1568                 if (mask)
1569                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1570                                                      (32 - width));
1571                 break;
1572         case RTE_FLOW_FIELD_IPV6_DSCP:
1573                 info[idx] = (struct field_modify_info){1, 0,
1574                                         MLX5_MODI_OUT_IP_DSCP};
1575                 if (mask)
1576                         mask[idx] = 0x3f >> (6 - width);
1577                 break;
1578         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1579                 info[idx] = (struct field_modify_info){1, 0,
1580                                         MLX5_MODI_OUT_IPV6_HOPLIMIT};
1581                 if (mask)
1582                         mask[idx] = 0xff >> (8 - width);
1583                 break;
1584         case RTE_FLOW_FIELD_IPV6_SRC:
1585                 if (mask) {
1586                         if (data->offset < 32) {
1587                                 info[idx] = (struct field_modify_info){4,
1588                                                 4 * idx,
1589                                                 MLX5_MODI_OUT_SIPV6_31_0};
1590                                 if (width < 32) {
1591                                         mask[idx] =
1592                                                 rte_cpu_to_be_32(0xffffffff >>
1593                                                                  (32 - width));
1594                                         width = 0;
1595                                 } else {
1596                                         mask[idx] = RTE_BE32(0xffffffff);
1597                                         width -= 32;
1598                                 }
1599                                 if (!width)
1600                                         break;
1601                                 ++idx;
1602                         }
1603                         if (data->offset < 64) {
1604                                 info[idx] = (struct field_modify_info){4,
1605                                                 4 * idx,
1606                                                 MLX5_MODI_OUT_SIPV6_63_32};
1607                                 if (width < 32) {
1608                                         mask[idx] =
1609                                                 rte_cpu_to_be_32(0xffffffff >>
1610                                                                  (32 - width));
1611                                         width = 0;
1612                                 } else {
1613                                         mask[idx] = RTE_BE32(0xffffffff);
1614                                         width -= 32;
1615                                 }
1616                                 if (!width)
1617                                         break;
1618                                 ++idx;
1619                         }
1620                         if (data->offset < 96) {
1621                                 info[idx] = (struct field_modify_info){4,
1622                                                 4 * idx,
1623                                                 MLX5_MODI_OUT_SIPV6_95_64};
1624                                 if (width < 32) {
1625                                         mask[idx] =
1626                                                 rte_cpu_to_be_32(0xffffffff >>
1627                                                                  (32 - width));
1628                                         width = 0;
1629                                 } else {
1630                                         mask[idx] = RTE_BE32(0xffffffff);
1631                                         width -= 32;
1632                                 }
1633                                 if (!width)
1634                                         break;
1635                                 ++idx;
1636                         }
1637                         info[idx] = (struct field_modify_info){4, 4 * idx,
1638                                                 MLX5_MODI_OUT_SIPV6_127_96};
1639                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1640                                                      (32 - width));
1641                 } else {
1642                         if (data->offset < 32)
1643                                 info[idx++] = (struct field_modify_info){4, 0,
1644                                                 MLX5_MODI_OUT_SIPV6_31_0};
1645                         if (data->offset < 64)
1646                                 info[idx++] = (struct field_modify_info){4, 0,
1647                                                 MLX5_MODI_OUT_SIPV6_63_32};
1648                         if (data->offset < 96)
1649                                 info[idx++] = (struct field_modify_info){4, 0,
1650                                                 MLX5_MODI_OUT_SIPV6_95_64};
1651                         if (data->offset < 128)
1652                                 info[idx++] = (struct field_modify_info){4, 0,
1653                                                 MLX5_MODI_OUT_SIPV6_127_96};
1654                 }
1655                 break;
1656         case RTE_FLOW_FIELD_IPV6_DST:
1657                 if (mask) {
1658                         if (data->offset < 32) {
1659                                 info[idx] = (struct field_modify_info){4,
1660                                                 4 * idx,
1661                                                 MLX5_MODI_OUT_DIPV6_31_0};
1662                                 if (width < 32) {
1663                                         mask[idx] =
1664                                                 rte_cpu_to_be_32(0xffffffff >>
1665                                                                  (32 - width));
1666                                         width = 0;
1667                                 } else {
1668                                         mask[idx] = RTE_BE32(0xffffffff);
1669                                         width -= 32;
1670                                 }
1671                                 if (!width)
1672                                         break;
1673                                 ++idx;
1674                         }
1675                         if (data->offset < 64) {
1676                                 info[idx] = (struct field_modify_info){4,
1677                                                 4 * idx,
1678                                                 MLX5_MODI_OUT_DIPV6_63_32};
1679                                 if (width < 32) {
1680                                         mask[idx] =
1681                                                 rte_cpu_to_be_32(0xffffffff >>
1682                                                                  (32 - width));
1683                                         width = 0;
1684                                 } else {
1685                                         mask[idx] = RTE_BE32(0xffffffff);
1686                                         width -= 32;
1687                                 }
1688                                 if (!width)
1689                                         break;
1690                                 ++idx;
1691                         }
1692                         if (data->offset < 96) {
1693                                 info[idx] = (struct field_modify_info){4,
1694                                                 4 * idx,
1695                                                 MLX5_MODI_OUT_DIPV6_95_64};
1696                                 if (width < 32) {
1697                                         mask[idx] =
1698                                                 rte_cpu_to_be_32(0xffffffff >>
1699                                                                  (32 - width));
1700                                         width = 0;
1701                                 } else {
1702                                         mask[idx] = RTE_BE32(0xffffffff);
1703                                         width -= 32;
1704                                 }
1705                                 if (!width)
1706                                         break;
1707                                 ++idx;
1708                         }
1709                         info[idx] = (struct field_modify_info){4, 4 * idx,
1710                                                 MLX5_MODI_OUT_DIPV6_127_96};
1711                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1712                                                      (32 - width));
1713                 } else {
1714                         if (data->offset < 32)
1715                                 info[idx++] = (struct field_modify_info){4, 0,
1716                                                 MLX5_MODI_OUT_DIPV6_31_0};
1717                         if (data->offset < 64)
1718                                 info[idx++] = (struct field_modify_info){4, 0,
1719                                                 MLX5_MODI_OUT_DIPV6_63_32};
1720                         if (data->offset < 96)
1721                                 info[idx++] = (struct field_modify_info){4, 0,
1722                                                 MLX5_MODI_OUT_DIPV6_95_64};
1723                         if (data->offset < 128)
1724                                 info[idx++] = (struct field_modify_info){4, 0,
1725                                                 MLX5_MODI_OUT_DIPV6_127_96};
1726                 }
1727                 break;
1728         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1729                 info[idx] = (struct field_modify_info){2, 0,
1730                                         MLX5_MODI_OUT_TCP_SPORT};
1731                 if (mask)
1732                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1733                 break;
1734         case RTE_FLOW_FIELD_TCP_PORT_DST:
1735                 info[idx] = (struct field_modify_info){2, 0,
1736                                         MLX5_MODI_OUT_TCP_DPORT};
1737                 if (mask)
1738                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1739                 break;
1740         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1741                 info[idx] = (struct field_modify_info){4, 0,
1742                                         MLX5_MODI_OUT_TCP_SEQ_NUM};
1743                 if (mask)
1744                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1745                                                      (32 - width));
1746                 break;
1747         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1748                 info[idx] = (struct field_modify_info){4, 0,
1749                                         MLX5_MODI_OUT_TCP_ACK_NUM};
1750                 if (mask)
1751                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1752                                                      (32 - width));
1753                 break;
1754         case RTE_FLOW_FIELD_TCP_FLAGS:
1755                 info[idx] = (struct field_modify_info){2, 0,
1756                                         MLX5_MODI_OUT_TCP_FLAGS};
1757                 if (mask)
1758                         mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
1759                 break;
1760         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1761                 info[idx] = (struct field_modify_info){2, 0,
1762                                         MLX5_MODI_OUT_UDP_SPORT};
1763                 if (mask)
1764                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1765                 break;
1766         case RTE_FLOW_FIELD_UDP_PORT_DST:
1767                 info[idx] = (struct field_modify_info){2, 0,
1768                                         MLX5_MODI_OUT_UDP_DPORT};
1769                 if (mask)
1770                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1771                 break;
1772         case RTE_FLOW_FIELD_VXLAN_VNI:
1773                 /* not supported yet */
1774                 break;
1775         case RTE_FLOW_FIELD_GENEVE_VNI:
1776                 /* not supported yet*/
1777                 break;
1778         case RTE_FLOW_FIELD_GTP_TEID:
1779                 info[idx] = (struct field_modify_info){4, 0,
1780                                         MLX5_MODI_GTP_TEID};
1781                 if (mask)
1782                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1783                                                      (32 - width));
1784                 break;
1785         case RTE_FLOW_FIELD_TAG:
1786                 {
1787                         int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
1788                                                    data->level, error);
1789                         if (reg < 0)
1790                                 return;
1791                         MLX5_ASSERT(reg != REG_NON);
1792                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1793                         info[idx] = (struct field_modify_info){4, 0,
1794                                                 reg_to_field[reg]};
1795                         if (mask)
1796                                 mask[idx] =
1797                                         rte_cpu_to_be_32(0xffffffff >>
1798                                                          (32 - width));
1799                 }
1800                 break;
1801         case RTE_FLOW_FIELD_MARK:
1802                 {
1803                         uint32_t mark_mask = priv->sh->dv_mark_mask;
1804                         uint32_t mark_count = __builtin_popcount(mark_mask);
1805                         int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
1806                                                        0, error);
1807                         if (reg < 0)
1808                                 return;
1809                         MLX5_ASSERT(reg != REG_NON);
1810                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1811                         info[idx] = (struct field_modify_info){4, 0,
1812                                                 reg_to_field[reg]};
1813                         if (mask)
1814                                 mask[idx] = rte_cpu_to_be_32((mark_mask >>
1815                                          (mark_count - width)) & mark_mask);
1816                 }
1817                 break;
1818         case RTE_FLOW_FIELD_META:
1819                 {
1820                         uint32_t meta_mask = priv->sh->dv_meta_mask;
1821                         uint32_t meta_count = __builtin_popcount(meta_mask);
1822                         uint32_t msk_c0 =
1823                                 rte_cpu_to_be_32(priv->sh->dv_regc0_mask);
1824                         uint32_t shl_c0 = rte_bsf32(msk_c0);
1825                         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1826                         if (reg < 0)
1827                                 return;
1828                         MLX5_ASSERT(reg != REG_NON);
1829                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1830                         if (reg == REG_C_0)
1831                                 *shift = shl_c0;
1832                         info[idx] = (struct field_modify_info){4, 0,
1833                                                 reg_to_field[reg]};
1834                         if (mask)
1835                                 mask[idx] = rte_cpu_to_be_32((meta_mask >>
1836                                         (meta_count - width)) & meta_mask);
1837                 }
1838                 break;
1839         case RTE_FLOW_FIELD_POINTER:
1840         case RTE_FLOW_FIELD_VALUE:
1841                 if (data->field == RTE_FLOW_FIELD_POINTER)
1842                         memcpy(&val, (void *)(uintptr_t)data->value,
1843                                sizeof(uint64_t));
1844                 else
1845                         val = data->value;
1846                 for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) {
1847                         if (mask[idx]) {
1848                                 if (dst_width == 48) {
1849                                         /*special case for MAC addresses */
1850                                         value[idx] = rte_cpu_to_be_16(val);
1851                                         val >>= 16;
1852                                         dst_width -= 16;
1853                                 } else if (dst_width > 16) {
1854                                         value[idx] = rte_cpu_to_be_32(val);
1855                                         val >>= 32;
1856                                 } else if (dst_width > 8) {
1857                                         value[idx] = rte_cpu_to_be_16(val);
1858                                         val >>= 16;
1859                                 } else {
1860                                         value[idx] = (uint8_t)val;
1861                                         val >>= 8;
1862                                 }
1863                                 if (*shift)
1864                                         value[idx] <<= *shift;
1865                                 if (!val)
1866                                         break;
1867                         }
1868                 }
1869                 break;
1870         default:
1871                 MLX5_ASSERT(false);
1872                 break;
1873         }
1874 }
1875
1876 /**
1877  * Convert modify_field action to DV specification.
1878  *
1879  * @param[in] dev
1880  *   Pointer to the rte_eth_dev structure.
1881  * @param[in,out] resource
1882  *   Pointer to the modify-header resource.
1883  * @param[in] action
1884  *   Pointer to action specification.
1885  * @param[in] attr
1886  *   Attributes of flow that includes this item.
1887  * @param[out] error
1888  *   Pointer to the error structure.
1889  *
1890  * @return
1891  *   0 on success, a negative errno value otherwise and rte_errno is set.
1892  */
1893 static int
1894 flow_dv_convert_action_modify_field
1895                         (struct rte_eth_dev *dev,
1896                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1897                          const struct rte_flow_action *action,
1898                          const struct rte_flow_attr *attr,
1899                          struct rte_flow_error *error)
1900 {
1901         struct mlx5_priv *priv = dev->data->dev_private;
1902         const struct rte_flow_action_modify_field *conf =
1903                 (const struct rte_flow_action_modify_field *)(action->conf);
1904         struct rte_flow_item item;
1905         struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
1906                                                                 {0, 0, 0} };
1907         struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
1908                                                                 {0, 0, 0} };
1909         uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1910         uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1911         uint32_t type;
1912         uint32_t shift = 0;
1913         uint32_t dst_width = mlx5_flow_item_field_width(priv, conf->dst.field);
1914
1915         if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
1916                 conf->src.field == RTE_FLOW_FIELD_VALUE) {
1917                 type = MLX5_MODIFICATION_TYPE_SET;
1918                 /** For SET fill the destination field (field) first. */
1919                 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
1920                                                   value, conf->width, dst_width,
1921                                                   &shift, dev, attr, error);
1922                 /** Then copy immediate value from source as per mask. */
1923                 mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask,
1924                                                   value, conf->width, dst_width,
1925                                                   &shift, dev, attr, error);
1926                 item.spec = &value;
1927         } else {
1928                 type = MLX5_MODIFICATION_TYPE_COPY;
1929                 /** For COPY fill the destination field (dcopy) without mask. */
1930                 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
1931                                                   value, conf->width, dst_width,
1932                                                   &shift, dev, attr, error);
1933                 /** Then construct the source field (field) with mask. */
1934                 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
1935                                                   value, conf->width, dst_width,
1936                                                   &shift, dev, attr, error);
1937         }
1938         item.mask = &mask;
1939         return flow_dv_convert_modify_action(&item,
1940                         field, dcopy, resource, type, error);
1941 }
1942
1943 /**
1944  * Validate MARK item.
1945  *
1946  * @param[in] dev
1947  *   Pointer to the rte_eth_dev structure.
1948  * @param[in] item
1949  *   Item specification.
1950  * @param[in] attr
1951  *   Attributes of flow that includes this item.
1952  * @param[out] error
1953  *   Pointer to error structure.
1954  *
1955  * @return
1956  *   0 on success, a negative errno value otherwise and rte_errno is set.
1957  */
1958 static int
1959 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1960                            const struct rte_flow_item *item,
1961                            const struct rte_flow_attr *attr __rte_unused,
1962                            struct rte_flow_error *error)
1963 {
1964         struct mlx5_priv *priv = dev->data->dev_private;
1965         struct mlx5_dev_config *config = &priv->config;
1966         const struct rte_flow_item_mark *spec = item->spec;
1967         const struct rte_flow_item_mark *mask = item->mask;
1968         const struct rte_flow_item_mark nic_mask = {
1969                 .id = priv->sh->dv_mark_mask,
1970         };
1971         int ret;
1972
1973         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1974                 return rte_flow_error_set(error, ENOTSUP,
1975                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1976                                           "extended metadata feature"
1977                                           " isn't enabled");
1978         if (!mlx5_flow_ext_mreg_supported(dev))
1979                 return rte_flow_error_set(error, ENOTSUP,
1980                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1981                                           "extended metadata register"
1982                                           " isn't supported");
1983         if (!nic_mask.id)
1984                 return rte_flow_error_set(error, ENOTSUP,
1985                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1986                                           "extended metadata register"
1987                                           " isn't available");
1988         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1989         if (ret < 0)
1990                 return ret;
1991         if (!spec)
1992                 return rte_flow_error_set(error, EINVAL,
1993                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1994                                           item->spec,
1995                                           "data cannot be empty");
1996         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1997                 return rte_flow_error_set(error, EINVAL,
1998                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1999                                           &spec->id,
2000                                           "mark id exceeds the limit");
2001         if (!mask)
2002                 mask = &nic_mask;
2003         if (!mask->id)
2004                 return rte_flow_error_set(error, EINVAL,
2005                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2006                                         "mask cannot be zero");
2007
2008         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2009                                         (const uint8_t *)&nic_mask,
2010                                         sizeof(struct rte_flow_item_mark),
2011                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2012         if (ret < 0)
2013                 return ret;
2014         return 0;
2015 }
2016
2017 /**
2018  * Validate META item.
2019  *
2020  * @param[in] dev
2021  *   Pointer to the rte_eth_dev structure.
2022  * @param[in] item
2023  *   Item specification.
2024  * @param[in] attr
2025  *   Attributes of flow that includes this item.
2026  * @param[out] error
2027  *   Pointer to error structure.
2028  *
2029  * @return
2030  *   0 on success, a negative errno value otherwise and rte_errno is set.
2031  */
2032 static int
2033 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
2034                            const struct rte_flow_item *item,
2035                            const struct rte_flow_attr *attr,
2036                            struct rte_flow_error *error)
2037 {
2038         struct mlx5_priv *priv = dev->data->dev_private;
2039         struct mlx5_dev_config *config = &priv->config;
2040         const struct rte_flow_item_meta *spec = item->spec;
2041         const struct rte_flow_item_meta *mask = item->mask;
2042         struct rte_flow_item_meta nic_mask = {
2043                 .data = UINT32_MAX
2044         };
2045         int reg;
2046         int ret;
2047
2048         if (!spec)
2049                 return rte_flow_error_set(error, EINVAL,
2050                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2051                                           item->spec,
2052                                           "data cannot be empty");
2053         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2054                 if (!mlx5_flow_ext_mreg_supported(dev))
2055                         return rte_flow_error_set(error, ENOTSUP,
2056                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2057                                           "extended metadata register"
2058                                           " isn't supported");
2059                 reg = flow_dv_get_metadata_reg(dev, attr, error);
2060                 if (reg < 0)
2061                         return reg;
2062                 if (reg == REG_NON)
2063                         return rte_flow_error_set(error, ENOTSUP,
2064                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2065                                         "unavalable extended metadata register");
2066                 if (reg == REG_B)
2067                         return rte_flow_error_set(error, ENOTSUP,
2068                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2069                                           "match on reg_b "
2070                                           "isn't supported");
2071                 if (reg != REG_A)
2072                         nic_mask.data = priv->sh->dv_meta_mask;
2073         } else {
2074                 if (attr->transfer)
2075                         return rte_flow_error_set(error, ENOTSUP,
2076                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2077                                         "extended metadata feature "
2078                                         "should be enabled when "
2079                                         "meta item is requested "
2080                                         "with e-switch mode ");
2081                 if (attr->ingress)
2082                         return rte_flow_error_set(error, ENOTSUP,
2083                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2084                                         "match on metadata for ingress "
2085                                         "is not supported in legacy "
2086                                         "metadata mode");
2087         }
2088         if (!mask)
2089                 mask = &rte_flow_item_meta_mask;
2090         if (!mask->data)
2091                 return rte_flow_error_set(error, EINVAL,
2092                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2093                                         "mask cannot be zero");
2094
2095         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2096                                         (const uint8_t *)&nic_mask,
2097                                         sizeof(struct rte_flow_item_meta),
2098                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2099         return ret;
2100 }
2101
2102 /**
2103  * Validate TAG item.
2104  *
2105  * @param[in] dev
2106  *   Pointer to the rte_eth_dev structure.
2107  * @param[in] item
2108  *   Item specification.
2109  * @param[in] attr
2110  *   Attributes of flow that includes this item.
2111  * @param[out] error
2112  *   Pointer to error structure.
2113  *
2114  * @return
2115  *   0 on success, a negative errno value otherwise and rte_errno is set.
2116  */
2117 static int
2118 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
2119                           const struct rte_flow_item *item,
2120                           const struct rte_flow_attr *attr __rte_unused,
2121                           struct rte_flow_error *error)
2122 {
2123         const struct rte_flow_item_tag *spec = item->spec;
2124         const struct rte_flow_item_tag *mask = item->mask;
2125         const struct rte_flow_item_tag nic_mask = {
2126                 .data = RTE_BE32(UINT32_MAX),
2127                 .index = 0xff,
2128         };
2129         int ret;
2130
2131         if (!mlx5_flow_ext_mreg_supported(dev))
2132                 return rte_flow_error_set(error, ENOTSUP,
2133                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2134                                           "extensive metadata register"
2135                                           " isn't supported");
2136         if (!spec)
2137                 return rte_flow_error_set(error, EINVAL,
2138                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2139                                           item->spec,
2140                                           "data cannot be empty");
2141         if (!mask)
2142                 mask = &rte_flow_item_tag_mask;
2143         if (!mask->data)
2144                 return rte_flow_error_set(error, EINVAL,
2145                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2146                                         "mask cannot be zero");
2147
2148         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2149                                         (const uint8_t *)&nic_mask,
2150                                         sizeof(struct rte_flow_item_tag),
2151                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2152         if (ret < 0)
2153                 return ret;
2154         if (mask->index != 0xff)
2155                 return rte_flow_error_set(error, EINVAL,
2156                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2157                                           "partial mask for tag index"
2158                                           " is not supported");
2159         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
2160         if (ret < 0)
2161                 return ret;
2162         MLX5_ASSERT(ret != REG_NON);
2163         return 0;
2164 }
2165
2166 /**
2167  * Validate vport item.
2168  *
2169  * @param[in] dev
2170  *   Pointer to the rte_eth_dev structure.
2171  * @param[in] item
2172  *   Item specification.
2173  * @param[in] attr
2174  *   Attributes of flow that includes this item.
2175  * @param[in] item_flags
2176  *   Bit-fields that holds the items detected until now.
2177  * @param[out] error
2178  *   Pointer to error structure.
2179  *
2180  * @return
2181  *   0 on success, a negative errno value otherwise and rte_errno is set.
2182  */
2183 static int
2184 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
2185                               const struct rte_flow_item *item,
2186                               const struct rte_flow_attr *attr,
2187                               uint64_t item_flags,
2188                               struct rte_flow_error *error)
2189 {
2190         const struct rte_flow_item_port_id *spec = item->spec;
2191         const struct rte_flow_item_port_id *mask = item->mask;
2192         const struct rte_flow_item_port_id switch_mask = {
2193                         .id = 0xffffffff,
2194         };
2195         struct mlx5_priv *esw_priv;
2196         struct mlx5_priv *dev_priv;
2197         int ret;
2198
2199         if (!attr->transfer)
2200                 return rte_flow_error_set(error, EINVAL,
2201                                           RTE_FLOW_ERROR_TYPE_ITEM,
2202                                           NULL,
2203                                           "match on port id is valid only"
2204                                           " when transfer flag is enabled");
2205         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
2206                 return rte_flow_error_set(error, ENOTSUP,
2207                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2208                                           "multiple source ports are not"
2209                                           " supported");
2210         if (!mask)
2211                 mask = &switch_mask;
2212         if (mask->id != 0xffffffff)
2213                 return rte_flow_error_set(error, ENOTSUP,
2214                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2215                                            mask,
2216                                            "no support for partial mask on"
2217                                            " \"id\" field");
2218         ret = mlx5_flow_item_acceptable
2219                                 (item, (const uint8_t *)mask,
2220                                  (const uint8_t *)&rte_flow_item_port_id_mask,
2221                                  sizeof(struct rte_flow_item_port_id),
2222                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2223         if (ret)
2224                 return ret;
2225         if (!spec)
2226                 return 0;
2227         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
2228         if (!esw_priv)
2229                 return rte_flow_error_set(error, rte_errno,
2230                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2231                                           "failed to obtain E-Switch info for"
2232                                           " port");
2233         dev_priv = mlx5_dev_to_eswitch_info(dev);
2234         if (!dev_priv)
2235                 return rte_flow_error_set(error, rte_errno,
2236                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2237                                           NULL,
2238                                           "failed to obtain E-Switch info");
2239         if (esw_priv->domain_id != dev_priv->domain_id)
2240                 return rte_flow_error_set(error, EINVAL,
2241                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2242                                           "cannot match on a port from a"
2243                                           " different E-Switch");
2244         return 0;
2245 }
2246
2247 /**
2248  * Validate VLAN item.
2249  *
2250  * @param[in] item
2251  *   Item specification.
2252  * @param[in] item_flags
2253  *   Bit-fields that holds the items detected until now.
2254  * @param[in] dev
2255  *   Ethernet device flow is being created on.
2256  * @param[out] error
2257  *   Pointer to error structure.
2258  *
2259  * @return
2260  *   0 on success, a negative errno value otherwise and rte_errno is set.
2261  */
2262 static int
2263 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
2264                            uint64_t item_flags,
2265                            struct rte_eth_dev *dev,
2266                            struct rte_flow_error *error)
2267 {
2268         const struct rte_flow_item_vlan *mask = item->mask;
2269         const struct rte_flow_item_vlan nic_mask = {
2270                 .tci = RTE_BE16(UINT16_MAX),
2271                 .inner_type = RTE_BE16(UINT16_MAX),
2272                 .has_more_vlan = 1,
2273         };
2274         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2275         int ret;
2276         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2277                                         MLX5_FLOW_LAYER_INNER_L4) :
2278                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2279                                         MLX5_FLOW_LAYER_OUTER_L4);
2280         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2281                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2282
2283         if (item_flags & vlanm)
2284                 return rte_flow_error_set(error, EINVAL,
2285                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2286                                           "multiple VLAN layers not supported");
2287         else if ((item_flags & l34m) != 0)
2288                 return rte_flow_error_set(error, EINVAL,
2289                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2290                                           "VLAN cannot follow L3/L4 layer");
2291         if (!mask)
2292                 mask = &rte_flow_item_vlan_mask;
2293         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2294                                         (const uint8_t *)&nic_mask,
2295                                         sizeof(struct rte_flow_item_vlan),
2296                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2297         if (ret)
2298                 return ret;
2299         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2300                 struct mlx5_priv *priv = dev->data->dev_private;
2301
2302                 if (priv->vmwa_context) {
2303                         /*
2304                          * Non-NULL context means we have a virtual machine
2305                          * and SR-IOV enabled, we have to create VLAN interface
2306                          * to make hypervisor to setup E-Switch vport
2307                          * context correctly. We avoid creating the multiple
2308                          * VLAN interfaces, so we cannot support VLAN tag mask.
2309                          */
2310                         return rte_flow_error_set(error, EINVAL,
2311                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2312                                                   item,
2313                                                   "VLAN tag mask is not"
2314                                                   " supported in virtual"
2315                                                   " environment");
2316                 }
2317         }
2318         return 0;
2319 }
2320
2321 /*
2322  * GTP flags are contained in 1 byte of the format:
2323  * -------------------------------------------
2324  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
2325  * |-----------------------------------------|
2326  * | value | Version | PT | Res | E | S | PN |
2327  * -------------------------------------------
2328  *
2329  * Matching is supported only for GTP flags E, S, PN.
2330  */
2331 #define MLX5_GTP_FLAGS_MASK     0x07
2332
2333 /**
2334  * Validate GTP item.
2335  *
2336  * @param[in] dev
2337  *   Pointer to the rte_eth_dev structure.
2338  * @param[in] item
2339  *   Item specification.
2340  * @param[in] item_flags
2341  *   Bit-fields that holds the items detected until now.
2342  * @param[out] error
2343  *   Pointer to error structure.
2344  *
2345  * @return
2346  *   0 on success, a negative errno value otherwise and rte_errno is set.
2347  */
2348 static int
2349 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
2350                           const struct rte_flow_item *item,
2351                           uint64_t item_flags,
2352                           struct rte_flow_error *error)
2353 {
2354         struct mlx5_priv *priv = dev->data->dev_private;
2355         const struct rte_flow_item_gtp *spec = item->spec;
2356         const struct rte_flow_item_gtp *mask = item->mask;
2357         const struct rte_flow_item_gtp nic_mask = {
2358                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
2359                 .msg_type = 0xff,
2360                 .teid = RTE_BE32(0xffffffff),
2361         };
2362
2363         if (!priv->config.hca_attr.tunnel_stateless_gtp)
2364                 return rte_flow_error_set(error, ENOTSUP,
2365                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2366                                           "GTP support is not enabled");
2367         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2368                 return rte_flow_error_set(error, ENOTSUP,
2369                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2370                                           "multiple tunnel layers not"
2371                                           " supported");
2372         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2373                 return rte_flow_error_set(error, EINVAL,
2374                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2375                                           "no outer UDP layer found");
2376         if (!mask)
2377                 mask = &rte_flow_item_gtp_mask;
2378         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
2379                 return rte_flow_error_set(error, ENOTSUP,
2380                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2381                                           "Match is supported for GTP"
2382                                           " flags only");
2383         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2384                                          (const uint8_t *)&nic_mask,
2385                                          sizeof(struct rte_flow_item_gtp),
2386                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2387 }
2388
2389 /**
2390  * Validate GTP PSC item.
2391  *
2392  * @param[in] item
2393  *   Item specification.
2394  * @param[in] last_item
2395  *   Previous validated item in the pattern items.
2396  * @param[in] gtp_item
2397  *   Previous GTP item specification.
2398  * @param[in] attr
2399  *   Pointer to flow attributes.
2400  * @param[out] error
2401  *   Pointer to error structure.
2402  *
2403  * @return
2404  *   0 on success, a negative errno value otherwise and rte_errno is set.
2405  */
2406 static int
2407 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
2408                               uint64_t last_item,
2409                               const struct rte_flow_item *gtp_item,
2410                               const struct rte_flow_attr *attr,
2411                               struct rte_flow_error *error)
2412 {
2413         const struct rte_flow_item_gtp *gtp_spec;
2414         const struct rte_flow_item_gtp *gtp_mask;
2415         const struct rte_flow_item_gtp_psc *mask;
2416         const struct rte_flow_item_gtp_psc nic_mask = {
2417                 .hdr.type = 0xF,
2418                 .hdr.qfi = 0x3F,
2419         };
2420
2421         if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
2422                 return rte_flow_error_set
2423                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2424                          "GTP PSC item must be preceded with GTP item");
2425         gtp_spec = gtp_item->spec;
2426         gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
2427         /* GTP spec and E flag is requested to match zero. */
2428         if (gtp_spec &&
2429                 (gtp_mask->v_pt_rsv_flags &
2430                 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
2431                 return rte_flow_error_set
2432                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2433                          "GTP E flag must be 1 to match GTP PSC");
2434         /* Check the flow is not created in group zero. */
2435         if (!attr->transfer && !attr->group)
2436                 return rte_flow_error_set
2437                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2438                          "GTP PSC is not supported for group 0");
2439         /* GTP spec is here and E flag is requested to match zero. */
2440         if (!item->spec)
2441                 return 0;
2442         mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
2443         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2444                                          (const uint8_t *)&nic_mask,
2445                                          sizeof(struct rte_flow_item_gtp_psc),
2446                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2447 }
2448
2449 /**
2450  * Validate IPV4 item.
2451  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
2452  * add specific validation of fragment_offset field,
2453  *
2454  * @param[in] item
2455  *   Item specification.
2456  * @param[in] item_flags
2457  *   Bit-fields that holds the items detected until now.
2458  * @param[out] error
2459  *   Pointer to error structure.
2460  *
2461  * @return
2462  *   0 on success, a negative errno value otherwise and rte_errno is set.
2463  */
2464 static int
2465 flow_dv_validate_item_ipv4(struct rte_eth_dev *dev,
2466                            const struct rte_flow_item *item,
2467                            uint64_t item_flags, uint64_t last_item,
2468                            uint16_t ether_type, struct rte_flow_error *error)
2469 {
2470         int ret;
2471         struct mlx5_priv *priv = dev->data->dev_private;
2472         const struct rte_flow_item_ipv4 *spec = item->spec;
2473         const struct rte_flow_item_ipv4 *last = item->last;
2474         const struct rte_flow_item_ipv4 *mask = item->mask;
2475         rte_be16_t fragment_offset_spec = 0;
2476         rte_be16_t fragment_offset_last = 0;
2477         struct rte_flow_item_ipv4 nic_ipv4_mask = {
2478                 .hdr = {
2479                         .src_addr = RTE_BE32(0xffffffff),
2480                         .dst_addr = RTE_BE32(0xffffffff),
2481                         .type_of_service = 0xff,
2482                         .fragment_offset = RTE_BE16(0xffff),
2483                         .next_proto_id = 0xff,
2484                         .time_to_live = 0xff,
2485                 },
2486         };
2487
2488         if (mask && (mask->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK)) {
2489                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2490                 bool ihl_cap = !tunnel ? priv->config.hca_attr.outer_ipv4_ihl :
2491                                priv->config.hca_attr.inner_ipv4_ihl;
2492                 if (!ihl_cap)
2493                         return rte_flow_error_set(error, ENOTSUP,
2494                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2495                                                   item,
2496                                                   "IPV4 ihl offload not supported");
2497                 nic_ipv4_mask.hdr.version_ihl = mask->hdr.version_ihl;
2498         }
2499         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
2500                                            ether_type, &nic_ipv4_mask,
2501                                            MLX5_ITEM_RANGE_ACCEPTED, error);
2502         if (ret < 0)
2503                 return ret;
2504         if (spec && mask)
2505                 fragment_offset_spec = spec->hdr.fragment_offset &
2506                                        mask->hdr.fragment_offset;
2507         if (!fragment_offset_spec)
2508                 return 0;
2509         /*
2510          * spec and mask are valid, enforce using full mask to make sure the
2511          * complete value is used correctly.
2512          */
2513         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2514                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2515                 return rte_flow_error_set(error, EINVAL,
2516                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2517                                           item, "must use full mask for"
2518                                           " fragment_offset");
2519         /*
2520          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
2521          * indicating this is 1st fragment of fragmented packet.
2522          * This is not yet supported in MLX5, return appropriate error message.
2523          */
2524         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
2525                 return rte_flow_error_set(error, ENOTSUP,
2526                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2527                                           "match on first fragment not "
2528                                           "supported");
2529         if (fragment_offset_spec && !last)
2530                 return rte_flow_error_set(error, ENOTSUP,
2531                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2532                                           "specified value not supported");
2533         /* spec and last are valid, validate the specified range. */
2534         fragment_offset_last = last->hdr.fragment_offset &
2535                                mask->hdr.fragment_offset;
2536         /*
2537          * Match on fragment_offset spec 0x2001 and last 0x3fff
2538          * means MF is 1 and frag-offset is > 0.
2539          * This packet is fragment 2nd and onward, excluding last.
2540          * This is not yet supported in MLX5, return appropriate
2541          * error message.
2542          */
2543         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
2544             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2545                 return rte_flow_error_set(error, ENOTSUP,
2546                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2547                                           last, "match on following "
2548                                           "fragments not supported");
2549         /*
2550          * Match on fragment_offset spec 0x0001 and last 0x1fff
2551          * means MF is 0 and frag-offset is > 0.
2552          * This packet is last fragment of fragmented packet.
2553          * This is not yet supported in MLX5, return appropriate
2554          * error message.
2555          */
2556         if (fragment_offset_spec == RTE_BE16(1) &&
2557             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
2558                 return rte_flow_error_set(error, ENOTSUP,
2559                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2560                                           last, "match on last "
2561                                           "fragment not supported");
2562         /*
2563          * Match on fragment_offset spec 0x0001 and last 0x3fff
2564          * means MF and/or frag-offset is not 0.
2565          * This is a fragmented packet.
2566          * Other range values are invalid and rejected.
2567          */
2568         if (!(fragment_offset_spec == RTE_BE16(1) &&
2569               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
2570                 return rte_flow_error_set(error, ENOTSUP,
2571                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2572                                           "specified range not supported");
2573         return 0;
2574 }
2575
2576 /**
2577  * Validate IPV6 fragment extension item.
2578  *
2579  * @param[in] item
2580  *   Item specification.
2581  * @param[in] item_flags
2582  *   Bit-fields that holds the items detected until now.
2583  * @param[out] error
2584  *   Pointer to error structure.
2585  *
2586  * @return
2587  *   0 on success, a negative errno value otherwise and rte_errno is set.
2588  */
2589 static int
2590 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
2591                                     uint64_t item_flags,
2592                                     struct rte_flow_error *error)
2593 {
2594         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
2595         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
2596         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
2597         rte_be16_t frag_data_spec = 0;
2598         rte_be16_t frag_data_last = 0;
2599         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2600         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2601                                       MLX5_FLOW_LAYER_OUTER_L4;
2602         int ret = 0;
2603         struct rte_flow_item_ipv6_frag_ext nic_mask = {
2604                 .hdr = {
2605                         .next_header = 0xff,
2606                         .frag_data = RTE_BE16(0xffff),
2607                 },
2608         };
2609
2610         if (item_flags & l4m)
2611                 return rte_flow_error_set(error, EINVAL,
2612                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2613                                           "ipv6 fragment extension item cannot "
2614                                           "follow L4 item.");
2615         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
2616             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2617                 return rte_flow_error_set(error, EINVAL,
2618                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2619                                           "ipv6 fragment extension item must "
2620                                           "follow ipv6 item");
2621         if (spec && mask)
2622                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
2623         if (!frag_data_spec)
2624                 return 0;
2625         /*
2626          * spec and mask are valid, enforce using full mask to make sure the
2627          * complete value is used correctly.
2628          */
2629         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
2630                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2631                 return rte_flow_error_set(error, EINVAL,
2632                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2633                                           item, "must use full mask for"
2634                                           " frag_data");
2635         /*
2636          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
2637          * This is 1st fragment of fragmented packet.
2638          */
2639         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
2640                 return rte_flow_error_set(error, ENOTSUP,
2641                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2642                                           "match on first fragment not "
2643                                           "supported");
2644         if (frag_data_spec && !last)
2645                 return rte_flow_error_set(error, EINVAL,
2646                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2647                                           "specified value not supported");
2648         ret = mlx5_flow_item_acceptable
2649                                 (item, (const uint8_t *)mask,
2650                                  (const uint8_t *)&nic_mask,
2651                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
2652                                  MLX5_ITEM_RANGE_ACCEPTED, error);
2653         if (ret)
2654                 return ret;
2655         /* spec and last are valid, validate the specified range. */
2656         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
2657         /*
2658          * Match on frag_data spec 0x0009 and last 0xfff9
2659          * means M is 1 and frag-offset is > 0.
2660          * This packet is fragment 2nd and onward, excluding last.
2661          * This is not yet supported in MLX5, return appropriate
2662          * error message.
2663          */
2664         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2665                                        RTE_IPV6_EHDR_MF_MASK) &&
2666             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2667                 return rte_flow_error_set(error, ENOTSUP,
2668                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2669                                           last, "match on following "
2670                                           "fragments not supported");
2671         /*
2672          * Match on frag_data spec 0x0008 and last 0xfff8
2673          * means M is 0 and frag-offset is > 0.
2674          * This packet is last fragment of fragmented packet.
2675          * This is not yet supported in MLX5, return appropriate
2676          * error message.
2677          */
2678         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2679             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2680                 return rte_flow_error_set(error, ENOTSUP,
2681                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2682                                           last, "match on last "
2683                                           "fragment not supported");
2684         /* Other range values are invalid and rejected. */
2685         return rte_flow_error_set(error, EINVAL,
2686                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2687                                   "specified range not supported");
2688 }
2689
2690 /*
2691  * Validate ASO CT item.
2692  *
2693  * @param[in] dev
2694  *   Pointer to the rte_eth_dev structure.
2695  * @param[in] item
2696  *   Item specification.
2697  * @param[in] item_flags
2698  *   Pointer to bit-fields that holds the items detected until now.
2699  * @param[out] error
2700  *   Pointer to error structure.
2701  *
2702  * @return
2703  *   0 on success, a negative errno value otherwise and rte_errno is set.
2704  */
2705 static int
2706 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
2707                              const struct rte_flow_item *item,
2708                              uint64_t *item_flags,
2709                              struct rte_flow_error *error)
2710 {
2711         const struct rte_flow_item_conntrack *spec = item->spec;
2712         const struct rte_flow_item_conntrack *mask = item->mask;
2713         RTE_SET_USED(dev);
2714         uint32_t flags;
2715
2716         if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
2717                 return rte_flow_error_set(error, EINVAL,
2718                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2719                                           "Only one CT is supported");
2720         if (!mask)
2721                 mask = &rte_flow_item_conntrack_mask;
2722         flags = spec->flags & mask->flags;
2723         if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
2724             ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
2725              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
2726              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
2727                 return rte_flow_error_set(error, EINVAL,
2728                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2729                                           "Conflict status bits");
2730         /* State change also needs to be considered. */
2731         *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
2732         return 0;
2733 }
2734
2735 /**
2736  * Validate the pop VLAN action.
2737  *
2738  * @param[in] dev
2739  *   Pointer to the rte_eth_dev structure.
2740  * @param[in] action_flags
2741  *   Holds the actions detected until now.
2742  * @param[in] action
2743  *   Pointer to the pop vlan action.
2744  * @param[in] item_flags
2745  *   The items found in this flow rule.
2746  * @param[in] attr
2747  *   Pointer to flow attributes.
2748  * @param[out] error
2749  *   Pointer to error structure.
2750  *
2751  * @return
2752  *   0 on success, a negative errno value otherwise and rte_errno is set.
2753  */
2754 static int
2755 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2756                                  uint64_t action_flags,
2757                                  const struct rte_flow_action *action,
2758                                  uint64_t item_flags,
2759                                  const struct rte_flow_attr *attr,
2760                                  struct rte_flow_error *error)
2761 {
2762         const struct mlx5_priv *priv = dev->data->dev_private;
2763         struct mlx5_dev_ctx_shared *sh = priv->sh;
2764         bool direction_error = false;
2765
2766         if (!priv->sh->pop_vlan_action)
2767                 return rte_flow_error_set(error, ENOTSUP,
2768                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2769                                           NULL,
2770                                           "pop vlan action is not supported");
2771         /* Pop VLAN is not supported in egress except for CX6 FDB mode. */
2772         if (attr->transfer) {
2773                 bool fdb_tx = priv->representor_id != UINT16_MAX;
2774                 bool is_cx5 = sh->steering_format_version ==
2775                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
2776
2777                 if (fdb_tx && is_cx5)
2778                         direction_error = true;
2779         } else if (attr->egress) {
2780                 direction_error = true;
2781         }
2782         if (direction_error)
2783                 return rte_flow_error_set(error, ENOTSUP,
2784                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2785                                           NULL,
2786                                           "pop vlan action not supported for egress");
2787         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2788                 return rte_flow_error_set(error, ENOTSUP,
2789                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2790                                           "no support for multiple VLAN "
2791                                           "actions");
2792         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2793         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2794             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2795                 return rte_flow_error_set(error, ENOTSUP,
2796                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2797                                           NULL,
2798                                           "cannot pop vlan after decap without "
2799                                           "match on inner vlan in the flow");
2800         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2801         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2802             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2803                 return rte_flow_error_set(error, ENOTSUP,
2804                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2805                                           NULL,
2806                                           "cannot pop vlan without a "
2807                                           "match on (outer) vlan in the flow");
2808         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2809                 return rte_flow_error_set(error, EINVAL,
2810                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2811                                           "wrong action order, port_id should "
2812                                           "be after pop VLAN action");
2813         if (!attr->transfer && priv->representor)
2814                 return rte_flow_error_set(error, ENOTSUP,
2815                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2816                                           "pop vlan action for VF representor "
2817                                           "not supported on NIC table");
2818         return 0;
2819 }
2820
2821 /**
2822  * Get VLAN default info from vlan match info.
2823  *
2824  * @param[in] items
2825  *   the list of item specifications.
2826  * @param[out] vlan
2827  *   pointer VLAN info to fill to.
2828  *
2829  * @return
2830  *   0 on success, a negative errno value otherwise and rte_errno is set.
2831  */
2832 static void
2833 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2834                                   struct rte_vlan_hdr *vlan)
2835 {
2836         const struct rte_flow_item_vlan nic_mask = {
2837                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2838                                 MLX5DV_FLOW_VLAN_VID_MASK),
2839                 .inner_type = RTE_BE16(0xffff),
2840         };
2841
2842         if (items == NULL)
2843                 return;
2844         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2845                 int type = items->type;
2846
2847                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2848                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2849                         break;
2850         }
2851         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2852                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2853                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2854
2855                 /* If VLAN item in pattern doesn't contain data, return here. */
2856                 if (!vlan_v)
2857                         return;
2858                 if (!vlan_m)
2859                         vlan_m = &nic_mask;
2860                 /* Only full match values are accepted */
2861                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2862                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2863                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2864                         vlan->vlan_tci |=
2865                                 rte_be_to_cpu_16(vlan_v->tci &
2866                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2867                 }
2868                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2869                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2870                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2871                         vlan->vlan_tci |=
2872                                 rte_be_to_cpu_16(vlan_v->tci &
2873                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2874                 }
2875                 if (vlan_m->inner_type == nic_mask.inner_type)
2876                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2877                                                            vlan_m->inner_type);
2878         }
2879 }
2880
2881 /**
2882  * Validate the push VLAN action.
2883  *
2884  * @param[in] dev
2885  *   Pointer to the rte_eth_dev structure.
2886  * @param[in] action_flags
2887  *   Holds the actions detected until now.
2888  * @param[in] item_flags
2889  *   The items found in this flow rule.
2890  * @param[in] action
2891  *   Pointer to the action structure.
2892  * @param[in] attr
2893  *   Pointer to flow attributes
2894  * @param[out] error
2895  *   Pointer to error structure.
2896  *
2897  * @return
2898  *   0 on success, a negative errno value otherwise and rte_errno is set.
2899  */
2900 static int
2901 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2902                                   uint64_t action_flags,
2903                                   const struct rte_flow_item_vlan *vlan_m,
2904                                   const struct rte_flow_action *action,
2905                                   const struct rte_flow_attr *attr,
2906                                   struct rte_flow_error *error)
2907 {
2908         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2909         const struct mlx5_priv *priv = dev->data->dev_private;
2910         struct mlx5_dev_ctx_shared *sh = priv->sh;
2911         bool direction_error = false;
2912
2913         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2914             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2915                 return rte_flow_error_set(error, EINVAL,
2916                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2917                                           "invalid vlan ethertype");
2918         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2919                 return rte_flow_error_set(error, EINVAL,
2920                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2921                                           "wrong action order, port_id should "
2922                                           "be after push VLAN");
2923         /* Push VLAN is not supported in ingress except for CX6 FDB mode. */
2924         if (attr->transfer) {
2925                 bool fdb_tx = priv->representor_id != UINT16_MAX;
2926                 bool is_cx5 = sh->steering_format_version ==
2927                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
2928
2929                 if (!fdb_tx && is_cx5)
2930                         direction_error = true;
2931         } else if (attr->ingress) {
2932                 direction_error = true;
2933         }
2934         if (direction_error)
2935                 return rte_flow_error_set(error, ENOTSUP,
2936                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2937                                           NULL,
2938                                           "push vlan action not supported for ingress");
2939         if (!attr->transfer && priv->representor)
2940                 return rte_flow_error_set(error, ENOTSUP,
2941                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2942                                           "push vlan action for VF representor "
2943                                           "not supported on NIC table");
2944         if (vlan_m &&
2945             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2946             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2947                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2948             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2949             !(mlx5_flow_find_action
2950                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2951                 return rte_flow_error_set(error, EINVAL,
2952                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2953                                           "not full match mask on VLAN PCP and "
2954                                           "there is no of_set_vlan_pcp action, "
2955                                           "push VLAN action cannot figure out "
2956                                           "PCP value");
2957         if (vlan_m &&
2958             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2959             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2960                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2961             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2962             !(mlx5_flow_find_action
2963                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2964                 return rte_flow_error_set(error, EINVAL,
2965                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2966                                           "not full match mask on VLAN VID and "
2967                                           "there is no of_set_vlan_vid action, "
2968                                           "push VLAN action cannot figure out "
2969                                           "VID value");
2970         (void)attr;
2971         return 0;
2972 }
2973
2974 /**
2975  * Validate the set VLAN PCP.
2976  *
2977  * @param[in] action_flags
2978  *   Holds the actions detected until now.
2979  * @param[in] actions
2980  *   Pointer to the list of actions remaining in the flow rule.
2981  * @param[out] error
2982  *   Pointer to error structure.
2983  *
2984  * @return
2985  *   0 on success, a negative errno value otherwise and rte_errno is set.
2986  */
2987 static int
2988 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2989                                      const struct rte_flow_action actions[],
2990                                      struct rte_flow_error *error)
2991 {
2992         const struct rte_flow_action *action = actions;
2993         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2994
2995         if (conf->vlan_pcp > 7)
2996                 return rte_flow_error_set(error, EINVAL,
2997                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2998                                           "VLAN PCP value is too big");
2999         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
3000                 return rte_flow_error_set(error, ENOTSUP,
3001                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3002                                           "set VLAN PCP action must follow "
3003                                           "the push VLAN action");
3004         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
3005                 return rte_flow_error_set(error, ENOTSUP,
3006                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3007                                           "Multiple VLAN PCP modification are "
3008                                           "not supported");
3009         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3010                 return rte_flow_error_set(error, EINVAL,
3011                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3012                                           "wrong action order, port_id should "
3013                                           "be after set VLAN PCP");
3014         return 0;
3015 }
3016
3017 /**
3018  * Validate the set VLAN VID.
3019  *
3020  * @param[in] item_flags
3021  *   Holds the items detected in this rule.
3022  * @param[in] action_flags
3023  *   Holds the actions detected until now.
3024  * @param[in] actions
3025  *   Pointer to the list of actions remaining in the flow rule.
3026  * @param[out] error
3027  *   Pointer to error structure.
3028  *
3029  * @return
3030  *   0 on success, a negative errno value otherwise and rte_errno is set.
3031  */
3032 static int
3033 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
3034                                      uint64_t action_flags,
3035                                      const struct rte_flow_action actions[],
3036                                      struct rte_flow_error *error)
3037 {
3038         const struct rte_flow_action *action = actions;
3039         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
3040
3041         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
3042                 return rte_flow_error_set(error, EINVAL,
3043                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3044                                           "VLAN VID value is too big");
3045         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
3046             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
3047                 return rte_flow_error_set(error, ENOTSUP,
3048                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3049                                           "set VLAN VID action must follow push"
3050                                           " VLAN action or match on VLAN item");
3051         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
3052                 return rte_flow_error_set(error, ENOTSUP,
3053                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3054                                           "Multiple VLAN VID modifications are "
3055                                           "not supported");
3056         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3057                 return rte_flow_error_set(error, EINVAL,
3058                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3059                                           "wrong action order, port_id should "
3060                                           "be after set VLAN VID");
3061         return 0;
3062 }
3063
3064 /*
3065  * Validate the FLAG action.
3066  *
3067  * @param[in] dev
3068  *   Pointer to the rte_eth_dev structure.
3069  * @param[in] action_flags
3070  *   Holds the actions detected until now.
3071  * @param[in] attr
3072  *   Pointer to flow attributes
3073  * @param[out] error
3074  *   Pointer to error structure.
3075  *
3076  * @return
3077  *   0 on success, a negative errno value otherwise and rte_errno is set.
3078  */
3079 static int
3080 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
3081                              uint64_t action_flags,
3082                              const struct rte_flow_attr *attr,
3083                              struct rte_flow_error *error)
3084 {
3085         struct mlx5_priv *priv = dev->data->dev_private;
3086         struct mlx5_dev_config *config = &priv->config;
3087         int ret;
3088
3089         /* Fall back if no extended metadata register support. */
3090         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3091                 return mlx5_flow_validate_action_flag(action_flags, attr,
3092                                                       error);
3093         /* Extensive metadata mode requires registers. */
3094         if (!mlx5_flow_ext_mreg_supported(dev))
3095                 return rte_flow_error_set(error, ENOTSUP,
3096                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3097                                           "no metadata registers "
3098                                           "to support flag action");
3099         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
3100                 return rte_flow_error_set(error, ENOTSUP,
3101                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3102                                           "extended metadata register"
3103                                           " isn't available");
3104         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3105         if (ret < 0)
3106                 return ret;
3107         MLX5_ASSERT(ret > 0);
3108         if (action_flags & MLX5_FLOW_ACTION_MARK)
3109                 return rte_flow_error_set(error, EINVAL,
3110                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3111                                           "can't mark and flag in same flow");
3112         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3113                 return rte_flow_error_set(error, EINVAL,
3114                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3115                                           "can't have 2 flag"
3116                                           " actions in same flow");
3117         return 0;
3118 }
3119
3120 /**
3121  * Validate MARK action.
3122  *
3123  * @param[in] dev
3124  *   Pointer to the rte_eth_dev structure.
3125  * @param[in] action
3126  *   Pointer to action.
3127  * @param[in] action_flags
3128  *   Holds the actions detected until now.
3129  * @param[in] attr
3130  *   Pointer to flow attributes
3131  * @param[out] error
3132  *   Pointer to error structure.
3133  *
3134  * @return
3135  *   0 on success, a negative errno value otherwise and rte_errno is set.
3136  */
3137 static int
3138 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
3139                              const struct rte_flow_action *action,
3140                              uint64_t action_flags,
3141                              const struct rte_flow_attr *attr,
3142                              struct rte_flow_error *error)
3143 {
3144         struct mlx5_priv *priv = dev->data->dev_private;
3145         struct mlx5_dev_config *config = &priv->config;
3146         const struct rte_flow_action_mark *mark = action->conf;
3147         int ret;
3148
3149         if (is_tunnel_offload_active(dev))
3150                 return rte_flow_error_set(error, ENOTSUP,
3151                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3152                                           "no mark action "
3153                                           "if tunnel offload active");
3154         /* Fall back if no extended metadata register support. */
3155         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3156                 return mlx5_flow_validate_action_mark(action, action_flags,
3157                                                       attr, error);
3158         /* Extensive metadata mode requires registers. */
3159         if (!mlx5_flow_ext_mreg_supported(dev))
3160                 return rte_flow_error_set(error, ENOTSUP,
3161                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3162                                           "no metadata registers "
3163                                           "to support mark action");
3164         if (!priv->sh->dv_mark_mask)
3165                 return rte_flow_error_set(error, ENOTSUP,
3166                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3167                                           "extended metadata register"
3168                                           " isn't available");
3169         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3170         if (ret < 0)
3171                 return ret;
3172         MLX5_ASSERT(ret > 0);
3173         if (!mark)
3174                 return rte_flow_error_set(error, EINVAL,
3175                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3176                                           "configuration cannot be null");
3177         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
3178                 return rte_flow_error_set(error, EINVAL,
3179                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3180                                           &mark->id,
3181                                           "mark id exceeds the limit");
3182         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3183                 return rte_flow_error_set(error, EINVAL,
3184                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3185                                           "can't flag and mark in same flow");
3186         if (action_flags & MLX5_FLOW_ACTION_MARK)
3187                 return rte_flow_error_set(error, EINVAL,
3188                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3189                                           "can't have 2 mark actions in same"
3190                                           " flow");
3191         return 0;
3192 }
3193
3194 /**
3195  * Validate SET_META action.
3196  *
3197  * @param[in] dev
3198  *   Pointer to the rte_eth_dev structure.
3199  * @param[in] action
3200  *   Pointer to the action structure.
3201  * @param[in] action_flags
3202  *   Holds the actions detected until now.
3203  * @param[in] attr
3204  *   Pointer to flow attributes
3205  * @param[out] error
3206  *   Pointer to error structure.
3207  *
3208  * @return
3209  *   0 on success, a negative errno value otherwise and rte_errno is set.
3210  */
3211 static int
3212 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
3213                                  const struct rte_flow_action *action,
3214                                  uint64_t action_flags __rte_unused,
3215                                  const struct rte_flow_attr *attr,
3216                                  struct rte_flow_error *error)
3217 {
3218         const struct rte_flow_action_set_meta *conf;
3219         uint32_t nic_mask = UINT32_MAX;
3220         int reg;
3221
3222         if (!mlx5_flow_ext_mreg_supported(dev))
3223                 return rte_flow_error_set(error, ENOTSUP,
3224                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3225                                           "extended metadata register"
3226                                           " isn't supported");
3227         reg = flow_dv_get_metadata_reg(dev, attr, error);
3228         if (reg < 0)
3229                 return reg;
3230         if (reg == REG_NON)
3231                 return rte_flow_error_set(error, ENOTSUP,
3232                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3233                                           "unavalable extended metadata register");
3234         if (reg != REG_A && reg != REG_B) {
3235                 struct mlx5_priv *priv = dev->data->dev_private;
3236
3237                 nic_mask = priv->sh->dv_meta_mask;
3238         }
3239         if (!(action->conf))
3240                 return rte_flow_error_set(error, EINVAL,
3241                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3242                                           "configuration cannot be null");
3243         conf = (const struct rte_flow_action_set_meta *)action->conf;
3244         if (!conf->mask)
3245                 return rte_flow_error_set(error, EINVAL,
3246                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3247                                           "zero mask doesn't have any effect");
3248         if (conf->mask & ~nic_mask)
3249                 return rte_flow_error_set(error, EINVAL,
3250                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3251                                           "meta data must be within reg C0");
3252         return 0;
3253 }
3254
3255 /**
3256  * Validate SET_TAG action.
3257  *
3258  * @param[in] dev
3259  *   Pointer to the rte_eth_dev structure.
3260  * @param[in] action
3261  *   Pointer to the action structure.
3262  * @param[in] action_flags
3263  *   Holds the actions detected until now.
3264  * @param[in] attr
3265  *   Pointer to flow attributes
3266  * @param[out] error
3267  *   Pointer to error structure.
3268  *
3269  * @return
3270  *   0 on success, a negative errno value otherwise and rte_errno is set.
3271  */
3272 static int
3273 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
3274                                 const struct rte_flow_action *action,
3275                                 uint64_t action_flags,
3276                                 const struct rte_flow_attr *attr,
3277                                 struct rte_flow_error *error)
3278 {
3279         const struct rte_flow_action_set_tag *conf;
3280         const uint64_t terminal_action_flags =
3281                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
3282                 MLX5_FLOW_ACTION_RSS;
3283         int ret;
3284
3285         if (!mlx5_flow_ext_mreg_supported(dev))
3286                 return rte_flow_error_set(error, ENOTSUP,
3287                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3288                                           "extensive metadata register"
3289                                           " isn't supported");
3290         if (!(action->conf))
3291                 return rte_flow_error_set(error, EINVAL,
3292                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3293                                           "configuration cannot be null");
3294         conf = (const struct rte_flow_action_set_tag *)action->conf;
3295         if (!conf->mask)
3296                 return rte_flow_error_set(error, EINVAL,
3297                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3298                                           "zero mask doesn't have any effect");
3299         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
3300         if (ret < 0)
3301                 return ret;
3302         if (!attr->transfer && attr->ingress &&
3303             (action_flags & terminal_action_flags))
3304                 return rte_flow_error_set(error, EINVAL,
3305                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3306                                           "set_tag has no effect"
3307                                           " with terminal actions");
3308         return 0;
3309 }
3310
3311 /**
3312  * Check if action counter is shared by either old or new mechanism.
3313  *
3314  * @param[in] action
3315  *   Pointer to the action structure.
3316  *
3317  * @return
3318  *   True when counter is shared, false otherwise.
3319  */
3320 static inline bool
3321 is_shared_action_count(const struct rte_flow_action *action)
3322 {
3323         const struct rte_flow_action_count *count =
3324                         (const struct rte_flow_action_count *)action->conf;
3325
3326         if ((int)action->type == MLX5_RTE_FLOW_ACTION_TYPE_COUNT)
3327                 return true;
3328         return !!(count && count->shared);
3329 }
3330
3331 /**
3332  * Validate count action.
3333  *
3334  * @param[in] dev
3335  *   Pointer to rte_eth_dev structure.
3336  * @param[in] shared
3337  *   Indicator if action is shared.
3338  * @param[in] action_flags
3339  *   Holds the actions detected until now.
3340  * @param[out] error
3341  *   Pointer to error structure.
3342  *
3343  * @return
3344  *   0 on success, a negative errno value otherwise and rte_errno is set.
3345  */
3346 static int
3347 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
3348                               uint64_t action_flags,
3349                               struct rte_flow_error *error)
3350 {
3351         struct mlx5_priv *priv = dev->data->dev_private;
3352
3353         if (!priv->config.devx)
3354                 goto notsup_err;
3355         if (action_flags & MLX5_FLOW_ACTION_COUNT)
3356                 return rte_flow_error_set(error, EINVAL,
3357                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3358                                           "duplicate count actions set");
3359         if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
3360             !priv->sh->flow_hit_aso_en)
3361                 return rte_flow_error_set(error, EINVAL,
3362                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3363                                           "old age and shared count combination is not supported");
3364 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
3365         return 0;
3366 #endif
3367 notsup_err:
3368         return rte_flow_error_set
3369                       (error, ENOTSUP,
3370                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3371                        NULL,
3372                        "count action not supported");
3373 }
3374
3375 /**
3376  * Validate the L2 encap action.
3377  *
3378  * @param[in] dev
3379  *   Pointer to the rte_eth_dev structure.
3380  * @param[in] action_flags
3381  *   Holds the actions detected until now.
3382  * @param[in] action
3383  *   Pointer to the action structure.
3384  * @param[in] attr
3385  *   Pointer to flow attributes.
3386  * @param[out] error
3387  *   Pointer to error structure.
3388  *
3389  * @return
3390  *   0 on success, a negative errno value otherwise and rte_errno is set.
3391  */
3392 static int
3393 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
3394                                  uint64_t action_flags,
3395                                  const struct rte_flow_action *action,
3396                                  const struct rte_flow_attr *attr,
3397                                  struct rte_flow_error *error)
3398 {
3399         const struct mlx5_priv *priv = dev->data->dev_private;
3400
3401         if (!(action->conf))
3402                 return rte_flow_error_set(error, EINVAL,
3403                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3404                                           "configuration cannot be null");
3405         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3406                 return rte_flow_error_set(error, EINVAL,
3407                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3408                                           "can only have a single encap action "
3409                                           "in a flow");
3410         if (!attr->transfer && priv->representor)
3411                 return rte_flow_error_set(error, ENOTSUP,
3412                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3413                                           "encap action for VF representor "
3414                                           "not supported on NIC table");
3415         return 0;
3416 }
3417
3418 /**
3419  * Validate a decap action.
3420  *
3421  * @param[in] dev
3422  *   Pointer to the rte_eth_dev structure.
3423  * @param[in] action_flags
3424  *   Holds the actions detected until now.
3425  * @param[in] action
3426  *   Pointer to the action structure.
3427  * @param[in] item_flags
3428  *   Holds the items detected.
3429  * @param[in] attr
3430  *   Pointer to flow attributes
3431  * @param[out] error
3432  *   Pointer to error structure.
3433  *
3434  * @return
3435  *   0 on success, a negative errno value otherwise and rte_errno is set.
3436  */
3437 static int
3438 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
3439                               uint64_t action_flags,
3440                               const struct rte_flow_action *action,
3441                               const uint64_t item_flags,
3442                               const struct rte_flow_attr *attr,
3443                               struct rte_flow_error *error)
3444 {
3445         const struct mlx5_priv *priv = dev->data->dev_private;
3446
3447         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
3448             !priv->config.decap_en)
3449                 return rte_flow_error_set(error, ENOTSUP,
3450                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3451                                           "decap is not enabled");
3452         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
3453                 return rte_flow_error_set(error, ENOTSUP,
3454                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3455                                           action_flags &
3456                                           MLX5_FLOW_ACTION_DECAP ? "can only "
3457                                           "have a single decap action" : "decap "
3458                                           "after encap is not supported");
3459         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
3460                 return rte_flow_error_set(error, EINVAL,
3461                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3462                                           "can't have decap action after"
3463                                           " modify action");
3464         if (attr->egress)
3465                 return rte_flow_error_set(error, ENOTSUP,
3466                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
3467                                           NULL,
3468                                           "decap action not supported for "
3469                                           "egress");
3470         if (!attr->transfer && priv->representor)
3471                 return rte_flow_error_set(error, ENOTSUP,
3472                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3473                                           "decap action for VF representor "
3474                                           "not supported on NIC table");
3475         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
3476             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
3477                 return rte_flow_error_set(error, ENOTSUP,
3478                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3479                                 "VXLAN item should be present for VXLAN decap");
3480         return 0;
3481 }
3482
3483 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
3484
3485 /**
3486  * Validate the raw encap and decap actions.
3487  *
3488  * @param[in] dev
3489  *   Pointer to the rte_eth_dev structure.
3490  * @param[in] decap
3491  *   Pointer to the decap action.
3492  * @param[in] encap
3493  *   Pointer to the encap action.
3494  * @param[in] attr
3495  *   Pointer to flow attributes
3496  * @param[in/out] action_flags
3497  *   Holds the actions detected until now.
3498  * @param[out] actions_n
3499  *   pointer to the number of actions counter.
3500  * @param[in] action
3501  *   Pointer to the action structure.
3502  * @param[in] item_flags
3503  *   Holds the items detected.
3504  * @param[out] error
3505  *   Pointer to error structure.
3506  *
3507  * @return
3508  *   0 on success, a negative errno value otherwise and rte_errno is set.
3509  */
3510 static int
3511 flow_dv_validate_action_raw_encap_decap
3512         (struct rte_eth_dev *dev,
3513          const struct rte_flow_action_raw_decap *decap,
3514          const struct rte_flow_action_raw_encap *encap,
3515          const struct rte_flow_attr *attr, uint64_t *action_flags,
3516          int *actions_n, const struct rte_flow_action *action,
3517          uint64_t item_flags, struct rte_flow_error *error)
3518 {
3519         const struct mlx5_priv *priv = dev->data->dev_private;
3520         int ret;
3521
3522         if (encap && (!encap->size || !encap->data))
3523                 return rte_flow_error_set(error, EINVAL,
3524                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3525                                           "raw encap data cannot be empty");
3526         if (decap && encap) {
3527                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
3528                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3529                         /* L3 encap. */
3530                         decap = NULL;
3531                 else if (encap->size <=
3532                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3533                            decap->size >
3534                            MLX5_ENCAPSULATION_DECISION_SIZE)
3535                         /* L3 decap. */
3536                         encap = NULL;
3537                 else if (encap->size >
3538                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3539                            decap->size >
3540                            MLX5_ENCAPSULATION_DECISION_SIZE)
3541                         /* 2 L2 actions: encap and decap. */
3542                         ;
3543                 else
3544                         return rte_flow_error_set(error,
3545                                 ENOTSUP,
3546                                 RTE_FLOW_ERROR_TYPE_ACTION,
3547                                 NULL, "unsupported too small "
3548                                 "raw decap and too small raw "
3549                                 "encap combination");
3550         }
3551         if (decap) {
3552                 ret = flow_dv_validate_action_decap(dev, *action_flags, action,
3553                                                     item_flags, attr, error);
3554                 if (ret < 0)
3555                         return ret;
3556                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
3557                 ++(*actions_n);
3558         }
3559         if (encap) {
3560                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
3561                         return rte_flow_error_set(error, ENOTSUP,
3562                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3563                                                   NULL,
3564                                                   "small raw encap size");
3565                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
3566                         return rte_flow_error_set(error, EINVAL,
3567                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3568                                                   NULL,
3569                                                   "more than one encap action");
3570                 if (!attr->transfer && priv->representor)
3571                         return rte_flow_error_set
3572                                         (error, ENOTSUP,
3573                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3574                                          "encap action for VF representor "
3575                                          "not supported on NIC table");
3576                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
3577                 ++(*actions_n);
3578         }
3579         return 0;
3580 }
3581
3582 /*
3583  * Validate the ASO CT action.
3584  *
3585  * @param[in] dev
3586  *   Pointer to the rte_eth_dev structure.
3587  * @param[in] action_flags
3588  *   Holds the actions detected until now.
3589  * @param[in] item_flags
3590  *   The items found in this flow rule.
3591  * @param[in] attr
3592  *   Pointer to flow attributes.
3593  * @param[out] error
3594  *   Pointer to error structure.
3595  *
3596  * @return
3597  *   0 on success, a negative errno value otherwise and rte_errno is set.
3598  */
3599 static int
3600 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
3601                                uint64_t action_flags,
3602                                uint64_t item_flags,
3603                                const struct rte_flow_attr *attr,
3604                                struct rte_flow_error *error)
3605 {
3606         RTE_SET_USED(dev);
3607
3608         if (attr->group == 0 && !attr->transfer)
3609                 return rte_flow_error_set(error, ENOTSUP,
3610                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3611                                           NULL,
3612                                           "Only support non-root table");
3613         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
3614                 return rte_flow_error_set(error, ENOTSUP,
3615                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3616                                           "CT cannot follow a fate action");
3617         if ((action_flags & MLX5_FLOW_ACTION_METER) ||
3618             (action_flags & MLX5_FLOW_ACTION_AGE))
3619                 return rte_flow_error_set(error, EINVAL,
3620                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3621                                           "Only one ASO action is supported");
3622         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3623                 return rte_flow_error_set(error, EINVAL,
3624                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3625                                           "Encap cannot exist before CT");
3626         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3627                 return rte_flow_error_set(error, EINVAL,
3628                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3629                                           "Not a outer TCP packet");
3630         return 0;
3631 }
3632
3633 int
3634 flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
3635                              struct mlx5_list_entry *entry, void *cb_ctx)
3636 {
3637         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3638         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3639         struct mlx5_flow_dv_encap_decap_resource *resource;
3640
3641         resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
3642                                 entry);
3643         if (resource->reformat_type == ctx_resource->reformat_type &&
3644             resource->ft_type == ctx_resource->ft_type &&
3645             resource->flags == ctx_resource->flags &&
3646             resource->size == ctx_resource->size &&
3647             !memcmp((const void *)resource->buf,
3648                     (const void *)ctx_resource->buf,
3649                     resource->size))
3650                 return 0;
3651         return -1;
3652 }
3653
3654 struct mlx5_list_entry *
3655 flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
3656 {
3657         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3658         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3659         struct mlx5dv_dr_domain *domain;
3660         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3661         struct mlx5_flow_dv_encap_decap_resource *resource;
3662         uint32_t idx;
3663         int ret;
3664
3665         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3666                 domain = sh->fdb_domain;
3667         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3668                 domain = sh->rx_domain;
3669         else
3670                 domain = sh->tx_domain;
3671         /* Register new encap/decap resource. */
3672         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
3673         if (!resource) {
3674                 rte_flow_error_set(ctx->error, ENOMEM,
3675                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3676                                    "cannot allocate resource memory");
3677                 return NULL;
3678         }
3679         *resource = *ctx_resource;
3680         resource->idx = idx;
3681         ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain,
3682                                                               resource,
3683                                                              &resource->action);
3684         if (ret) {
3685                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
3686                 rte_flow_error_set(ctx->error, ENOMEM,
3687                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3688                                    NULL, "cannot create action");
3689                 return NULL;
3690         }
3691
3692         return &resource->entry;
3693 }
3694
3695 struct mlx5_list_entry *
3696 flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
3697                              void *cb_ctx)
3698 {
3699         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3700         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3701         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3702         uint32_t idx;
3703
3704         cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
3705                                            &idx);
3706         if (!cache_resource) {
3707                 rte_flow_error_set(ctx->error, ENOMEM,
3708                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3709                                    "cannot allocate resource memory");
3710                 return NULL;
3711         }
3712         memcpy(cache_resource, oentry, sizeof(*cache_resource));
3713         cache_resource->idx = idx;
3714         return &cache_resource->entry;
3715 }
3716
3717 void
3718 flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3719 {
3720         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3721         struct mlx5_flow_dv_encap_decap_resource *res =
3722                                        container_of(entry, typeof(*res), entry);
3723
3724         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
3725 }
3726
3727 /**
3728  * Find existing encap/decap resource or create and register a new one.
3729  *
3730  * @param[in, out] dev
3731  *   Pointer to rte_eth_dev structure.
3732  * @param[in, out] resource
3733  *   Pointer to encap/decap resource.
3734  * @parm[in, out] dev_flow
3735  *   Pointer to the dev_flow.
3736  * @param[out] error
3737  *   pointer to error structure.
3738  *
3739  * @return
3740  *   0 on success otherwise -errno and errno is set.
3741  */
3742 static int
3743 flow_dv_encap_decap_resource_register
3744                         (struct rte_eth_dev *dev,
3745                          struct mlx5_flow_dv_encap_decap_resource *resource,
3746                          struct mlx5_flow *dev_flow,
3747                          struct rte_flow_error *error)
3748 {
3749         struct mlx5_priv *priv = dev->data->dev_private;
3750         struct mlx5_dev_ctx_shared *sh = priv->sh;
3751         struct mlx5_list_entry *entry;
3752         union {
3753                 struct {
3754                         uint32_t ft_type:8;
3755                         uint32_t refmt_type:8;
3756                         /*
3757                          * Header reformat actions can be shared between
3758                          * non-root tables. One bit to indicate non-root
3759                          * table or not.
3760                          */
3761                         uint32_t is_root:1;
3762                         uint32_t reserve:15;
3763                 };
3764                 uint32_t v32;
3765         } encap_decap_key = {
3766                 {
3767                         .ft_type = resource->ft_type,
3768                         .refmt_type = resource->reformat_type,
3769                         .is_root = !!dev_flow->dv.group,
3770                         .reserve = 0,
3771                 }
3772         };
3773         struct mlx5_flow_cb_ctx ctx = {
3774                 .error = error,
3775                 .data = resource,
3776         };
3777         struct mlx5_hlist *encaps_decaps;
3778         uint64_t key64;
3779
3780         encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
3781                                 "encaps_decaps",
3782                                 MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
3783                                 true, true, sh,
3784                                 flow_dv_encap_decap_create_cb,
3785                                 flow_dv_encap_decap_match_cb,
3786                                 flow_dv_encap_decap_remove_cb,
3787                                 flow_dv_encap_decap_clone_cb,
3788                                 flow_dv_encap_decap_clone_free_cb);
3789         if (unlikely(!encaps_decaps))
3790                 return -rte_errno;
3791         resource->flags = dev_flow->dv.group ? 0 : 1;
3792         key64 =  __rte_raw_cksum(&encap_decap_key.v32,
3793                                  sizeof(encap_decap_key.v32), 0);
3794         if (resource->reformat_type !=
3795             MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
3796             resource->size)
3797                 key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
3798         entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
3799         if (!entry)
3800                 return -rte_errno;
3801         resource = container_of(entry, typeof(*resource), entry);
3802         dev_flow->dv.encap_decap = resource;
3803         dev_flow->handle->dvh.rix_encap_decap = resource->idx;
3804         return 0;
3805 }
3806
3807 /**
3808  * Find existing table jump resource or create and register a new one.
3809  *
3810  * @param[in, out] dev
3811  *   Pointer to rte_eth_dev structure.
3812  * @param[in, out] tbl
3813  *   Pointer to flow table resource.
3814  * @parm[in, out] dev_flow
3815  *   Pointer to the dev_flow.
3816  * @param[out] error
3817  *   pointer to error structure.
3818  *
3819  * @return
3820  *   0 on success otherwise -errno and errno is set.
3821  */
3822 static int
3823 flow_dv_jump_tbl_resource_register
3824                         (struct rte_eth_dev *dev __rte_unused,
3825                          struct mlx5_flow_tbl_resource *tbl,
3826                          struct mlx5_flow *dev_flow,
3827                          struct rte_flow_error *error __rte_unused)
3828 {
3829         struct mlx5_flow_tbl_data_entry *tbl_data =
3830                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
3831
3832         MLX5_ASSERT(tbl);
3833         MLX5_ASSERT(tbl_data->jump.action);
3834         dev_flow->handle->rix_jump = tbl_data->idx;
3835         dev_flow->dv.jump = &tbl_data->jump;
3836         return 0;
3837 }
3838
3839 int
3840 flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
3841                          struct mlx5_list_entry *entry, void *cb_ctx)
3842 {
3843         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3844         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3845         struct mlx5_flow_dv_port_id_action_resource *res =
3846                                        container_of(entry, typeof(*res), entry);
3847
3848         return ref->port_id != res->port_id;
3849 }
3850
3851 struct mlx5_list_entry *
3852 flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
3853 {
3854         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3855         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3856         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3857         struct mlx5_flow_dv_port_id_action_resource *resource;
3858         uint32_t idx;
3859         int ret;
3860
3861         /* Register new port id action resource. */
3862         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3863         if (!resource) {
3864                 rte_flow_error_set(ctx->error, ENOMEM,
3865                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3866                                    "cannot allocate port_id action memory");
3867                 return NULL;
3868         }
3869         *resource = *ref;
3870         ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
3871                                                         ref->port_id,
3872                                                         &resource->action);
3873         if (ret) {
3874                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
3875                 rte_flow_error_set(ctx->error, ENOMEM,
3876                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3877                                    "cannot create action");
3878                 return NULL;
3879         }
3880         resource->idx = idx;
3881         return &resource->entry;
3882 }
3883
3884 struct mlx5_list_entry *
3885 flow_dv_port_id_clone_cb(void *tool_ctx,
3886                          struct mlx5_list_entry *entry __rte_unused,
3887                          void *cb_ctx)
3888 {
3889         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3890         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3891         struct mlx5_flow_dv_port_id_action_resource *resource;
3892         uint32_t idx;
3893
3894         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3895         if (!resource) {
3896                 rte_flow_error_set(ctx->error, ENOMEM,
3897                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3898                                    "cannot allocate port_id action memory");
3899                 return NULL;
3900         }
3901         memcpy(resource, entry, sizeof(*resource));
3902         resource->idx = idx;
3903         return &resource->entry;
3904 }
3905
3906 void
3907 flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3908 {
3909         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3910         struct mlx5_flow_dv_port_id_action_resource *resource =
3911                                   container_of(entry, typeof(*resource), entry);
3912
3913         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
3914 }
3915
3916 /**
3917  * Find existing table port ID resource or create and register a new one.
3918  *
3919  * @param[in, out] dev
3920  *   Pointer to rte_eth_dev structure.
3921  * @param[in, out] ref
3922  *   Pointer to port ID action resource reference.
3923  * @parm[in, out] dev_flow
3924  *   Pointer to the dev_flow.
3925  * @param[out] error
3926  *   pointer to error structure.
3927  *
3928  * @return
3929  *   0 on success otherwise -errno and errno is set.
3930  */
3931 static int
3932 flow_dv_port_id_action_resource_register
3933                         (struct rte_eth_dev *dev,
3934                          struct mlx5_flow_dv_port_id_action_resource *ref,
3935                          struct mlx5_flow *dev_flow,
3936                          struct rte_flow_error *error)
3937 {
3938         struct mlx5_priv *priv = dev->data->dev_private;
3939         struct mlx5_list_entry *entry;
3940         struct mlx5_flow_dv_port_id_action_resource *resource;
3941         struct mlx5_flow_cb_ctx ctx = {
3942                 .error = error,
3943                 .data = ref,
3944         };
3945
3946         entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
3947         if (!entry)
3948                 return -rte_errno;
3949         resource = container_of(entry, typeof(*resource), entry);
3950         dev_flow->dv.port_id_action = resource;
3951         dev_flow->handle->rix_port_id_action = resource->idx;
3952         return 0;
3953 }
3954
3955 int
3956 flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
3957                            struct mlx5_list_entry *entry, void *cb_ctx)
3958 {
3959         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3960         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3961         struct mlx5_flow_dv_push_vlan_action_resource *res =
3962                                        container_of(entry, typeof(*res), entry);
3963
3964         return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
3965 }
3966
3967 struct mlx5_list_entry *
3968 flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
3969 {
3970         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3971         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3972         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3973         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3974         struct mlx5dv_dr_domain *domain;
3975         uint32_t idx;
3976         int ret;
3977
3978         /* Register new port id action resource. */
3979         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3980         if (!resource) {
3981                 rte_flow_error_set(ctx->error, ENOMEM,
3982                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3983                                    "cannot allocate push_vlan action memory");
3984                 return NULL;
3985         }
3986         *resource = *ref;
3987         if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3988                 domain = sh->fdb_domain;
3989         else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3990                 domain = sh->rx_domain;
3991         else
3992                 domain = sh->tx_domain;
3993         ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
3994                                                         &resource->action);
3995         if (ret) {
3996                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
3997                 rte_flow_error_set(ctx->error, ENOMEM,
3998                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3999                                    "cannot create push vlan action");
4000                 return NULL;
4001         }
4002         resource->idx = idx;
4003         return &resource->entry;
4004 }
4005
4006 struct mlx5_list_entry *
4007 flow_dv_push_vlan_clone_cb(void *tool_ctx,
4008                            struct mlx5_list_entry *entry __rte_unused,
4009                            void *cb_ctx)
4010 {
4011         struct mlx5_dev_ctx_shared *sh = tool_ctx;
4012         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4013         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4014         uint32_t idx;
4015
4016         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
4017         if (!resource) {
4018                 rte_flow_error_set(ctx->error, ENOMEM,
4019                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4020                                    "cannot allocate push_vlan action memory");
4021                 return NULL;
4022         }
4023         memcpy(resource, entry, sizeof(*resource));
4024         resource->idx = idx;
4025         return &resource->entry;
4026 }
4027
4028 void
4029 flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4030 {
4031         struct mlx5_dev_ctx_shared *sh = tool_ctx;
4032         struct mlx5_flow_dv_push_vlan_action_resource *resource =
4033                                   container_of(entry, typeof(*resource), entry);
4034
4035         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
4036 }
4037
4038 /**
4039  * Find existing push vlan resource or create and register a new one.
4040  *
4041  * @param [in, out] dev
4042  *   Pointer to rte_eth_dev structure.
4043  * @param[in, out] ref
4044  *   Pointer to port ID action resource reference.
4045  * @parm[in, out] dev_flow
4046  *   Pointer to the dev_flow.
4047  * @param[out] error
4048  *   pointer to error structure.
4049  *
4050  * @return
4051  *   0 on success otherwise -errno and errno is set.
4052  */
4053 static int
4054 flow_dv_push_vlan_action_resource_register
4055                        (struct rte_eth_dev *dev,
4056                         struct mlx5_flow_dv_push_vlan_action_resource *ref,
4057                         struct mlx5_flow *dev_flow,
4058                         struct rte_flow_error *error)
4059 {
4060         struct mlx5_priv *priv = dev->data->dev_private;
4061         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4062         struct mlx5_list_entry *entry;
4063         struct mlx5_flow_cb_ctx ctx = {
4064                 .error = error,
4065                 .data = ref,
4066         };
4067
4068         entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
4069         if (!entry)
4070                 return -rte_errno;
4071         resource = container_of(entry, typeof(*resource), entry);
4072
4073         dev_flow->handle->dvh.rix_push_vlan = resource->idx;
4074         dev_flow->dv.push_vlan_res = resource;
4075         return 0;
4076 }
4077
4078 /**
4079  * Get the size of specific rte_flow_item_type hdr size
4080  *
4081  * @param[in] item_type
4082  *   Tested rte_flow_item_type.
4083  *
4084  * @return
4085  *   sizeof struct item_type, 0 if void or irrelevant.
4086  */
4087 static size_t
4088 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
4089 {
4090         size_t retval;
4091
4092         switch (item_type) {
4093         case RTE_FLOW_ITEM_TYPE_ETH:
4094                 retval = sizeof(struct rte_ether_hdr);
4095                 break;
4096         case RTE_FLOW_ITEM_TYPE_VLAN:
4097                 retval = sizeof(struct rte_vlan_hdr);
4098                 break;
4099         case RTE_FLOW_ITEM_TYPE_IPV4:
4100                 retval = sizeof(struct rte_ipv4_hdr);
4101                 break;
4102         case RTE_FLOW_ITEM_TYPE_IPV6:
4103                 retval = sizeof(struct rte_ipv6_hdr);
4104                 break;
4105         case RTE_FLOW_ITEM_TYPE_UDP:
4106                 retval = sizeof(struct rte_udp_hdr);
4107                 break;
4108         case RTE_FLOW_ITEM_TYPE_TCP:
4109                 retval = sizeof(struct rte_tcp_hdr);
4110                 break;
4111         case RTE_FLOW_ITEM_TYPE_VXLAN:
4112         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4113                 retval = sizeof(struct rte_vxlan_hdr);
4114                 break;
4115         case RTE_FLOW_ITEM_TYPE_GRE:
4116         case RTE_FLOW_ITEM_TYPE_NVGRE:
4117                 retval = sizeof(struct rte_gre_hdr);
4118                 break;
4119         case RTE_FLOW_ITEM_TYPE_MPLS:
4120                 retval = sizeof(struct rte_mpls_hdr);
4121                 break;
4122         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
4123         default:
4124                 retval = 0;
4125                 break;
4126         }
4127         return retval;
4128 }
4129
4130 #define MLX5_ENCAP_IPV4_VERSION         0x40
4131 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
4132 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
4133 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
4134 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
4135 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
4136 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
4137
4138 /**
4139  * Convert the encap action data from list of rte_flow_item to raw buffer
4140  *
4141  * @param[in] items
4142  *   Pointer to rte_flow_item objects list.
4143  * @param[out] buf
4144  *   Pointer to the output buffer.
4145  * @param[out] size
4146  *   Pointer to the output buffer size.
4147  * @param[out] error
4148  *   Pointer to the error structure.
4149  *
4150  * @return
4151  *   0 on success, a negative errno value otherwise and rte_errno is set.
4152  */
4153 static int
4154 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
4155                            size_t *size, struct rte_flow_error *error)
4156 {
4157         struct rte_ether_hdr *eth = NULL;
4158         struct rte_vlan_hdr *vlan = NULL;
4159         struct rte_ipv4_hdr *ipv4 = NULL;
4160         struct rte_ipv6_hdr *ipv6 = NULL;
4161         struct rte_udp_hdr *udp = NULL;
4162         struct rte_vxlan_hdr *vxlan = NULL;
4163         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
4164         struct rte_gre_hdr *gre = NULL;
4165         size_t len;
4166         size_t temp_size = 0;
4167
4168         if (!items)
4169                 return rte_flow_error_set(error, EINVAL,
4170                                           RTE_FLOW_ERROR_TYPE_ACTION,
4171                                           NULL, "invalid empty data");
4172         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4173                 len = flow_dv_get_item_hdr_len(items->type);
4174                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
4175                         return rte_flow_error_set(error, EINVAL,
4176                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4177                                                   (void *)items->type,
4178                                                   "items total size is too big"
4179                                                   " for encap action");
4180                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
4181                 switch (items->type) {
4182                 case RTE_FLOW_ITEM_TYPE_ETH:
4183                         eth = (struct rte_ether_hdr *)&buf[temp_size];
4184                         break;
4185                 case RTE_FLOW_ITEM_TYPE_VLAN:
4186                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
4187                         if (!eth)
4188                                 return rte_flow_error_set(error, EINVAL,
4189                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4190                                                 (void *)items->type,
4191                                                 "eth header not found");
4192                         if (!eth->ether_type)
4193                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
4194                         break;
4195                 case RTE_FLOW_ITEM_TYPE_IPV4:
4196                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
4197                         if (!vlan && !eth)
4198                                 return rte_flow_error_set(error, EINVAL,
4199                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4200                                                 (void *)items->type,
4201                                                 "neither eth nor vlan"
4202                                                 " header found");
4203                         if (vlan && !vlan->eth_proto)
4204                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4205                         else if (eth && !eth->ether_type)
4206                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4207                         if (!ipv4->version_ihl)
4208                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
4209                                                     MLX5_ENCAP_IPV4_IHL_MIN;
4210                         if (!ipv4->time_to_live)
4211                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
4212                         break;
4213                 case RTE_FLOW_ITEM_TYPE_IPV6:
4214                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
4215                         if (!vlan && !eth)
4216                                 return rte_flow_error_set(error, EINVAL,
4217                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4218                                                 (void *)items->type,
4219                                                 "neither eth nor vlan"
4220                                                 " header found");
4221                         if (vlan && !vlan->eth_proto)
4222                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4223                         else if (eth && !eth->ether_type)
4224                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4225                         if (!ipv6->vtc_flow)
4226                                 ipv6->vtc_flow =
4227                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
4228                         if (!ipv6->hop_limits)
4229                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
4230                         break;
4231                 case RTE_FLOW_ITEM_TYPE_UDP:
4232                         udp = (struct rte_udp_hdr *)&buf[temp_size];
4233                         if (!ipv4 && !ipv6)
4234                                 return rte_flow_error_set(error, EINVAL,
4235                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4236                                                 (void *)items->type,
4237                                                 "ip header not found");
4238                         if (ipv4 && !ipv4->next_proto_id)
4239                                 ipv4->next_proto_id = IPPROTO_UDP;
4240                         else if (ipv6 && !ipv6->proto)
4241                                 ipv6->proto = IPPROTO_UDP;
4242                         break;
4243                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4244                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
4245                         if (!udp)
4246                                 return rte_flow_error_set(error, EINVAL,
4247                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4248                                                 (void *)items->type,
4249                                                 "udp header not found");
4250                         if (!udp->dst_port)
4251                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
4252                         if (!vxlan->vx_flags)
4253                                 vxlan->vx_flags =
4254                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
4255                         break;
4256                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4257                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
4258                         if (!udp)
4259                                 return rte_flow_error_set(error, EINVAL,
4260                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4261                                                 (void *)items->type,
4262                                                 "udp header not found");
4263                         if (!vxlan_gpe->proto)
4264                                 return rte_flow_error_set(error, EINVAL,
4265                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4266                                                 (void *)items->type,
4267                                                 "next protocol not found");
4268                         if (!udp->dst_port)
4269                                 udp->dst_port =
4270                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
4271                         if (!vxlan_gpe->vx_flags)
4272                                 vxlan_gpe->vx_flags =
4273                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
4274                         break;
4275                 case RTE_FLOW_ITEM_TYPE_GRE:
4276                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4277                         gre = (struct rte_gre_hdr *)&buf[temp_size];
4278                         if (!gre->proto)
4279                                 return rte_flow_error_set(error, EINVAL,
4280                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4281                                                 (void *)items->type,
4282                                                 "next protocol not found");
4283                         if (!ipv4 && !ipv6)
4284                                 return rte_flow_error_set(error, EINVAL,
4285                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4286                                                 (void *)items->type,
4287                                                 "ip header not found");
4288                         if (ipv4 && !ipv4->next_proto_id)
4289                                 ipv4->next_proto_id = IPPROTO_GRE;
4290                         else if (ipv6 && !ipv6->proto)
4291                                 ipv6->proto = IPPROTO_GRE;
4292                         break;
4293                 case RTE_FLOW_ITEM_TYPE_VOID:
4294                         break;
4295                 default:
4296                         return rte_flow_error_set(error, EINVAL,
4297                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4298                                                   (void *)items->type,
4299                                                   "unsupported item type");
4300                         break;
4301                 }
4302                 temp_size += len;
4303         }
4304         *size = temp_size;
4305         return 0;
4306 }
4307
4308 static int
4309 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
4310 {
4311         struct rte_ether_hdr *eth = NULL;
4312         struct rte_vlan_hdr *vlan = NULL;
4313         struct rte_ipv6_hdr *ipv6 = NULL;
4314         struct rte_udp_hdr *udp = NULL;
4315         char *next_hdr;
4316         uint16_t proto;
4317
4318         eth = (struct rte_ether_hdr *)data;
4319         next_hdr = (char *)(eth + 1);
4320         proto = RTE_BE16(eth->ether_type);
4321
4322         /* VLAN skipping */
4323         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
4324                 vlan = (struct rte_vlan_hdr *)next_hdr;
4325                 proto = RTE_BE16(vlan->eth_proto);
4326                 next_hdr += sizeof(struct rte_vlan_hdr);
4327         }
4328
4329         /* HW calculates IPv4 csum. no need to proceed */
4330         if (proto == RTE_ETHER_TYPE_IPV4)
4331                 return 0;
4332
4333         /* non IPv4/IPv6 header. not supported */
4334         if (proto != RTE_ETHER_TYPE_IPV6) {
4335                 return rte_flow_error_set(error, ENOTSUP,
4336                                           RTE_FLOW_ERROR_TYPE_ACTION,
4337                                           NULL, "Cannot offload non IPv4/IPv6");
4338         }
4339
4340         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
4341
4342         /* ignore non UDP */
4343         if (ipv6->proto != IPPROTO_UDP)
4344                 return 0;
4345
4346         udp = (struct rte_udp_hdr *)(ipv6 + 1);
4347         udp->dgram_cksum = 0;
4348
4349         return 0;
4350 }
4351
4352 /**
4353  * Convert L2 encap action to DV specification.
4354  *
4355  * @param[in] dev
4356  *   Pointer to rte_eth_dev structure.
4357  * @param[in] action
4358  *   Pointer to action structure.
4359  * @param[in, out] dev_flow
4360  *   Pointer to the mlx5_flow.
4361  * @param[in] transfer
4362  *   Mark if the flow is E-Switch flow.
4363  * @param[out] error
4364  *   Pointer to the error structure.
4365  *
4366  * @return
4367  *   0 on success, a negative errno value otherwise and rte_errno is set.
4368  */
4369 static int
4370 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
4371                                const struct rte_flow_action *action,
4372                                struct mlx5_flow *dev_flow,
4373                                uint8_t transfer,
4374                                struct rte_flow_error *error)
4375 {
4376         const struct rte_flow_item *encap_data;
4377         const struct rte_flow_action_raw_encap *raw_encap_data;
4378         struct mlx5_flow_dv_encap_decap_resource res = {
4379                 .reformat_type =
4380                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
4381                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4382                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
4383         };
4384
4385         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4386                 raw_encap_data =
4387                         (const struct rte_flow_action_raw_encap *)action->conf;
4388                 res.size = raw_encap_data->size;
4389                 memcpy(res.buf, raw_encap_data->data, res.size);
4390         } else {
4391                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
4392                         encap_data =
4393                                 ((const struct rte_flow_action_vxlan_encap *)
4394                                                 action->conf)->definition;
4395                 else
4396                         encap_data =
4397                                 ((const struct rte_flow_action_nvgre_encap *)
4398                                                 action->conf)->definition;
4399                 if (flow_dv_convert_encap_data(encap_data, res.buf,
4400                                                &res.size, error))
4401                         return -rte_errno;
4402         }
4403         if (flow_dv_zero_encap_udp_csum(res.buf, error))
4404                 return -rte_errno;
4405         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4406                 return rte_flow_error_set(error, EINVAL,
4407                                           RTE_FLOW_ERROR_TYPE_ACTION,
4408                                           NULL, "can't create L2 encap action");
4409         return 0;
4410 }
4411
4412 /**
4413  * Convert L2 decap action to DV specification.
4414  *
4415  * @param[in] dev
4416  *   Pointer to rte_eth_dev structure.
4417  * @param[in, out] dev_flow
4418  *   Pointer to the mlx5_flow.
4419  * @param[in] transfer
4420  *   Mark if the flow is E-Switch flow.
4421  * @param[out] error
4422  *   Pointer to the error structure.
4423  *
4424  * @return
4425  *   0 on success, a negative errno value otherwise and rte_errno is set.
4426  */
4427 static int
4428 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
4429                                struct mlx5_flow *dev_flow,
4430                                uint8_t transfer,
4431                                struct rte_flow_error *error)
4432 {
4433         struct mlx5_flow_dv_encap_decap_resource res = {
4434                 .size = 0,
4435                 .reformat_type =
4436                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
4437                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4438                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
4439         };
4440
4441         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4442                 return rte_flow_error_set(error, EINVAL,
4443                                           RTE_FLOW_ERROR_TYPE_ACTION,
4444                                           NULL, "can't create L2 decap action");
4445         return 0;
4446 }
4447
4448 /**
4449  * Convert raw decap/encap (L3 tunnel) action to DV specification.
4450  *
4451  * @param[in] dev
4452  *   Pointer to rte_eth_dev structure.
4453  * @param[in] action
4454  *   Pointer to action structure.
4455  * @param[in, out] dev_flow
4456  *   Pointer to the mlx5_flow.
4457  * @param[in] attr
4458  *   Pointer to the flow attributes.
4459  * @param[out] error
4460  *   Pointer to the error structure.
4461  *
4462  * @return
4463  *   0 on success, a negative errno value otherwise and rte_errno is set.
4464  */
4465 static int
4466 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
4467                                 const struct rte_flow_action *action,
4468                                 struct mlx5_flow *dev_flow,
4469                                 const struct rte_flow_attr *attr,
4470                                 struct rte_flow_error *error)
4471 {
4472         const struct rte_flow_action_raw_encap *encap_data;
4473         struct mlx5_flow_dv_encap_decap_resource res;
4474
4475         memset(&res, 0, sizeof(res));
4476         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
4477         res.size = encap_data->size;
4478         memcpy(res.buf, encap_data->data, res.size);
4479         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
4480                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
4481                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
4482         if (attr->transfer)
4483                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4484         else
4485                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4486                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4487         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4488                 return rte_flow_error_set(error, EINVAL,
4489                                           RTE_FLOW_ERROR_TYPE_ACTION,
4490                                           NULL, "can't create encap action");
4491         return 0;
4492 }
4493
4494 /**
4495  * Create action push VLAN.
4496  *
4497  * @param[in] dev
4498  *   Pointer to rte_eth_dev structure.
4499  * @param[in] attr
4500  *   Pointer to the flow attributes.
4501  * @param[in] vlan
4502  *   Pointer to the vlan to push to the Ethernet header.
4503  * @param[in, out] dev_flow
4504  *   Pointer to the mlx5_flow.
4505  * @param[out] error
4506  *   Pointer to the error structure.
4507  *
4508  * @return
4509  *   0 on success, a negative errno value otherwise and rte_errno is set.
4510  */
4511 static int
4512 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
4513                                 const struct rte_flow_attr *attr,
4514                                 const struct rte_vlan_hdr *vlan,
4515                                 struct mlx5_flow *dev_flow,
4516                                 struct rte_flow_error *error)
4517 {
4518         struct mlx5_flow_dv_push_vlan_action_resource res;
4519
4520         memset(&res, 0, sizeof(res));
4521         res.vlan_tag =
4522                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
4523                                  vlan->vlan_tci);
4524         if (attr->transfer)
4525                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4526         else
4527                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4528                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4529         return flow_dv_push_vlan_action_resource_register
4530                                             (dev, &res, dev_flow, error);
4531 }
4532
4533 /**
4534  * Validate the modify-header actions.
4535  *
4536  * @param[in] action_flags
4537  *   Holds the actions detected until now.
4538  * @param[in] action
4539  *   Pointer to the modify action.
4540  * @param[out] error
4541  *   Pointer to error structure.
4542  *
4543  * @return
4544  *   0 on success, a negative errno value otherwise and rte_errno is set.
4545  */
4546 static int
4547 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
4548                                    const struct rte_flow_action *action,
4549                                    struct rte_flow_error *error)
4550 {
4551         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
4552                 return rte_flow_error_set(error, EINVAL,
4553                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4554                                           NULL, "action configuration not set");
4555         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
4556                 return rte_flow_error_set(error, EINVAL,
4557                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4558                                           "can't have encap action before"
4559                                           " modify action");
4560         return 0;
4561 }
4562
4563 /**
4564  * Validate the modify-header MAC address actions.
4565  *
4566  * @param[in] action_flags
4567  *   Holds the actions detected until now.
4568  * @param[in] action
4569  *   Pointer to the modify action.
4570  * @param[in] item_flags
4571  *   Holds the items detected.
4572  * @param[out] error
4573  *   Pointer to error structure.
4574  *
4575  * @return
4576  *   0 on success, a negative errno value otherwise and rte_errno is set.
4577  */
4578 static int
4579 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
4580                                    const struct rte_flow_action *action,
4581                                    const uint64_t item_flags,
4582                                    struct rte_flow_error *error)
4583 {
4584         int ret = 0;
4585
4586         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4587         if (!ret) {
4588                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
4589                         return rte_flow_error_set(error, EINVAL,
4590                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4591                                                   NULL,
4592                                                   "no L2 item in pattern");
4593         }
4594         return ret;
4595 }
4596
4597 /**
4598  * Validate the modify-header IPv4 address actions.
4599  *
4600  * @param[in] action_flags
4601  *   Holds the actions detected until now.
4602  * @param[in] action
4603  *   Pointer to the modify action.
4604  * @param[in] item_flags
4605  *   Holds the items detected.
4606  * @param[out] error
4607  *   Pointer to error structure.
4608  *
4609  * @return
4610  *   0 on success, a negative errno value otherwise and rte_errno is set.
4611  */
4612 static int
4613 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
4614                                     const struct rte_flow_action *action,
4615                                     const uint64_t item_flags,
4616                                     struct rte_flow_error *error)
4617 {
4618         int ret = 0;
4619         uint64_t layer;
4620
4621         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4622         if (!ret) {
4623                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4624                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4625                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4626                 if (!(item_flags & layer))
4627                         return rte_flow_error_set(error, EINVAL,
4628                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4629                                                   NULL,
4630                                                   "no ipv4 item in pattern");
4631         }
4632         return ret;
4633 }
4634
4635 /**
4636  * Validate the modify-header IPv6 address actions.
4637  *
4638  * @param[in] action_flags
4639  *   Holds the actions detected until now.
4640  * @param[in] action
4641  *   Pointer to the modify action.
4642  * @param[in] item_flags
4643  *   Holds the items detected.
4644  * @param[out] error
4645  *   Pointer to error structure.
4646  *
4647  * @return
4648  *   0 on success, a negative errno value otherwise and rte_errno is set.
4649  */
4650 static int
4651 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
4652                                     const struct rte_flow_action *action,
4653                                     const uint64_t item_flags,
4654                                     struct rte_flow_error *error)
4655 {
4656         int ret = 0;
4657         uint64_t layer;
4658
4659         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4660         if (!ret) {
4661                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4662                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4663                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4664                 if (!(item_flags & layer))
4665                         return rte_flow_error_set(error, EINVAL,
4666                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4667                                                   NULL,
4668                                                   "no ipv6 item in pattern");
4669         }
4670         return ret;
4671 }
4672
4673 /**
4674  * Validate the modify-header TP actions.
4675  *
4676  * @param[in] action_flags
4677  *   Holds the actions detected until now.
4678  * @param[in] action
4679  *   Pointer to the modify action.
4680  * @param[in] item_flags
4681  *   Holds the items detected.
4682  * @param[out] error
4683  *   Pointer to error structure.
4684  *
4685  * @return
4686  *   0 on success, a negative errno value otherwise and rte_errno is set.
4687  */
4688 static int
4689 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
4690                                   const struct rte_flow_action *action,
4691                                   const uint64_t item_flags,
4692                                   struct rte_flow_error *error)
4693 {
4694         int ret = 0;
4695         uint64_t layer;
4696
4697         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4698         if (!ret) {
4699                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4700                                  MLX5_FLOW_LAYER_INNER_L4 :
4701                                  MLX5_FLOW_LAYER_OUTER_L4;
4702                 if (!(item_flags & layer))
4703                         return rte_flow_error_set(error, EINVAL,
4704                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4705                                                   NULL, "no transport layer "
4706                                                   "in pattern");
4707         }
4708         return ret;
4709 }
4710
4711 /**
4712  * Validate the modify-header actions of increment/decrement
4713  * TCP Sequence-number.
4714  *
4715  * @param[in] action_flags
4716  *   Holds the actions detected until now.
4717  * @param[in] action
4718  *   Pointer to the modify action.
4719  * @param[in] item_flags
4720  *   Holds the items detected.
4721  * @param[out] error
4722  *   Pointer to error structure.
4723  *
4724  * @return
4725  *   0 on success, a negative errno value otherwise and rte_errno is set.
4726  */
4727 static int
4728 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
4729                                        const struct rte_flow_action *action,
4730                                        const uint64_t item_flags,
4731                                        struct rte_flow_error *error)
4732 {
4733         int ret = 0;
4734         uint64_t layer;
4735
4736         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4737         if (!ret) {
4738                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4739                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4740                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4741                 if (!(item_flags & layer))
4742                         return rte_flow_error_set(error, EINVAL,
4743                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4744                                                   NULL, "no TCP item in"
4745                                                   " pattern");
4746                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
4747                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
4748                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
4749                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
4750                         return rte_flow_error_set(error, EINVAL,
4751                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4752                                                   NULL,
4753                                                   "cannot decrease and increase"
4754                                                   " TCP sequence number"
4755                                                   " at the same time");
4756         }
4757         return ret;
4758 }
4759
4760 /**
4761  * Validate the modify-header actions of increment/decrement
4762  * TCP Acknowledgment number.
4763  *
4764  * @param[in] action_flags
4765  *   Holds the actions detected until now.
4766  * @param[in] action
4767  *   Pointer to the modify action.
4768  * @param[in] item_flags
4769  *   Holds the items detected.
4770  * @param[out] error
4771  *   Pointer to error structure.
4772  *
4773  * @return
4774  *   0 on success, a negative errno value otherwise and rte_errno is set.
4775  */
4776 static int
4777 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
4778                                        const struct rte_flow_action *action,
4779                                        const uint64_t item_flags,
4780                                        struct rte_flow_error *error)
4781 {
4782         int ret = 0;
4783         uint64_t layer;
4784
4785         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4786         if (!ret) {
4787                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4788                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4789                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4790                 if (!(item_flags & layer))
4791                         return rte_flow_error_set(error, EINVAL,
4792                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4793                                                   NULL, "no TCP item in"
4794                                                   " pattern");
4795                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
4796                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
4797                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
4798                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
4799                         return rte_flow_error_set(error, EINVAL,
4800                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4801                                                   NULL,
4802                                                   "cannot decrease and increase"
4803                                                   " TCP acknowledgment number"
4804                                                   " at the same time");
4805         }
4806         return ret;
4807 }
4808
4809 /**
4810  * Validate the modify-header TTL actions.
4811  *
4812  * @param[in] action_flags
4813  *   Holds the actions detected until now.
4814  * @param[in] action
4815  *   Pointer to the modify action.
4816  * @param[in] item_flags
4817  *   Holds the items detected.
4818  * @param[out] error
4819  *   Pointer to error structure.
4820  *
4821  * @return
4822  *   0 on success, a negative errno value otherwise and rte_errno is set.
4823  */
4824 static int
4825 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
4826                                    const struct rte_flow_action *action,
4827                                    const uint64_t item_flags,
4828                                    struct rte_flow_error *error)
4829 {
4830         int ret = 0;
4831         uint64_t layer;
4832
4833         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4834         if (!ret) {
4835                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4836                                  MLX5_FLOW_LAYER_INNER_L3 :
4837                                  MLX5_FLOW_LAYER_OUTER_L3;
4838                 if (!(item_flags & layer))
4839                         return rte_flow_error_set(error, EINVAL,
4840                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4841                                                   NULL,
4842                                                   "no IP protocol in pattern");
4843         }
4844         return ret;
4845 }
4846
4847 /**
4848  * Validate the generic modify field actions.
4849  * @param[in] dev
4850  *   Pointer to the rte_eth_dev structure.
4851  * @param[in] action_flags
4852  *   Holds the actions detected until now.
4853  * @param[in] action
4854  *   Pointer to the modify action.
4855  * @param[in] attr
4856  *   Pointer to the flow attributes.
4857  * @param[out] error
4858  *   Pointer to error structure.
4859  *
4860  * @return
4861  *   Number of header fields to modify (0 or more) on success,
4862  *   a negative errno value otherwise and rte_errno is set.
4863  */
4864 static int
4865 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev,
4866                                    const uint64_t action_flags,
4867                                    const struct rte_flow_action *action,
4868                                    const struct rte_flow_attr *attr,
4869                                    struct rte_flow_error *error)
4870 {
4871         int ret = 0;
4872         struct mlx5_priv *priv = dev->data->dev_private;
4873         struct mlx5_dev_config *config = &priv->config;
4874         const struct rte_flow_action_modify_field *action_modify_field =
4875                 action->conf;
4876         uint32_t dst_width = mlx5_flow_item_field_width(priv,
4877                                 action_modify_field->dst.field);
4878         uint32_t src_width = mlx5_flow_item_field_width(priv,
4879                                 action_modify_field->src.field);
4880
4881         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4882         if (ret)
4883                 return ret;
4884
4885         if (action_modify_field->width == 0)
4886                 return rte_flow_error_set(error, EINVAL,
4887                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4888                                 "no bits are requested to be modified");
4889         else if (action_modify_field->width > dst_width ||
4890                  action_modify_field->width > src_width)
4891                 return rte_flow_error_set(error, EINVAL,
4892                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4893                                 "cannot modify more bits than"
4894                                 " the width of a field");
4895         if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE &&
4896             action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) {
4897                 if ((action_modify_field->dst.offset +
4898                      action_modify_field->width > dst_width) ||
4899                     (action_modify_field->dst.offset % 32))
4900                         return rte_flow_error_set(error, EINVAL,
4901                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4902                                         "destination offset is too big"
4903                                         " or not aligned to 4 bytes");
4904                 if (action_modify_field->dst.level &&
4905                     action_modify_field->dst.field != RTE_FLOW_FIELD_TAG)
4906                         return rte_flow_error_set(error, ENOTSUP,
4907                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4908                                         "inner header fields modification"
4909                                         " is not supported");
4910         }
4911         if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE &&
4912             action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) {
4913                 if (!attr->transfer && !attr->group)
4914                         return rte_flow_error_set(error, ENOTSUP,
4915                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4916                                         "modify field action is not"
4917                                         " supported for group 0");
4918                 if ((action_modify_field->src.offset +
4919                      action_modify_field->width > src_width) ||
4920                     (action_modify_field->src.offset % 32))
4921                         return rte_flow_error_set(error, EINVAL,
4922                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4923                                         "source offset is too big"
4924                                         " or not aligned to 4 bytes");
4925                 if (action_modify_field->src.level &&
4926                     action_modify_field->src.field != RTE_FLOW_FIELD_TAG)
4927                         return rte_flow_error_set(error, ENOTSUP,
4928                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4929                                         "inner header fields modification"
4930                                         " is not supported");
4931         }
4932         if ((action_modify_field->dst.field ==
4933              action_modify_field->src.field) &&
4934             (action_modify_field->dst.level ==
4935              action_modify_field->src.level))
4936                 return rte_flow_error_set(error, EINVAL,
4937                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4938                                 "source and destination fields"
4939                                 " cannot be the same");
4940         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE ||
4941             action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER ||
4942             action_modify_field->dst.field == RTE_FLOW_FIELD_MARK)
4943                 return rte_flow_error_set(error, EINVAL,
4944                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4945                                 "mark, immediate value or a pointer to it"
4946                                 " cannot be used as a destination");
4947         if (action_modify_field->dst.field == RTE_FLOW_FIELD_START ||
4948             action_modify_field->src.field == RTE_FLOW_FIELD_START)
4949                 return rte_flow_error_set(error, ENOTSUP,
4950                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4951                                 "modifications of an arbitrary"
4952                                 " place in a packet is not supported");
4953         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE ||
4954             action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE)
4955                 return rte_flow_error_set(error, ENOTSUP,
4956                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4957                                 "modifications of the 802.1Q Tag"
4958                                 " Identifier is not supported");
4959         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI ||
4960             action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI)
4961                 return rte_flow_error_set(error, ENOTSUP,
4962                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4963                                 "modifications of the VXLAN Network"
4964                                 " Identifier is not supported");
4965         if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI ||
4966             action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI)
4967                 return rte_flow_error_set(error, ENOTSUP,
4968                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4969                                 "modifications of the GENEVE Network"
4970                                 " Identifier is not supported");
4971         if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK ||
4972             action_modify_field->src.field == RTE_FLOW_FIELD_MARK ||
4973             action_modify_field->dst.field == RTE_FLOW_FIELD_META ||
4974             action_modify_field->src.field == RTE_FLOW_FIELD_META) {
4975                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4976                     !mlx5_flow_ext_mreg_supported(dev))
4977                         return rte_flow_error_set(error, ENOTSUP,
4978                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4979                                         "cannot modify mark or metadata without"
4980                                         " extended metadata register support");
4981         }
4982         if (action_modify_field->operation != RTE_FLOW_MODIFY_SET)
4983                 return rte_flow_error_set(error, ENOTSUP,
4984                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4985                                 "add and sub operations"
4986                                 " are not supported");
4987         return (action_modify_field->width / 32) +
4988                !!(action_modify_field->width % 32);
4989 }
4990
4991 /**
4992  * Validate jump action.
4993  *
4994  * @param[in] action
4995  *   Pointer to the jump action.
4996  * @param[in] action_flags
4997  *   Holds the actions detected until now.
4998  * @param[in] attributes
4999  *   Pointer to flow attributes
5000  * @param[in] external
5001  *   Action belongs to flow rule created by request external to PMD.
5002  * @param[out] error
5003  *   Pointer to error structure.
5004  *
5005  * @return
5006  *   0 on success, a negative errno value otherwise and rte_errno is set.
5007  */
5008 static int
5009 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
5010                              const struct mlx5_flow_tunnel *tunnel,
5011                              const struct rte_flow_action *action,
5012                              uint64_t action_flags,
5013                              const struct rte_flow_attr *attributes,
5014                              bool external, struct rte_flow_error *error)
5015 {
5016         uint32_t target_group, table;
5017         int ret = 0;
5018         struct flow_grp_info grp_info = {
5019                 .external = !!external,
5020                 .transfer = !!attributes->transfer,
5021                 .fdb_def_rule = 1,
5022                 .std_tbl_fix = 0
5023         };
5024         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
5025                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5026                 return rte_flow_error_set(error, EINVAL,
5027                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5028                                           "can't have 2 fate actions in"
5029                                           " same flow");
5030         if (!action->conf)
5031                 return rte_flow_error_set(error, EINVAL,
5032                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5033                                           NULL, "action configuration not set");
5034         target_group =
5035                 ((const struct rte_flow_action_jump *)action->conf)->group;
5036         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
5037                                        &grp_info, error);
5038         if (ret)
5039                 return ret;
5040         if (attributes->group == target_group &&
5041             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
5042                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
5043                 return rte_flow_error_set(error, EINVAL,
5044                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5045                                           "target group must be other than"
5046                                           " the current flow group");
5047         return 0;
5048 }
5049
5050 /*
5051  * Validate the port_id action.
5052  *
5053  * @param[in] dev
5054  *   Pointer to rte_eth_dev structure.
5055  * @param[in] action_flags
5056  *   Bit-fields that holds the actions detected until now.
5057  * @param[in] action
5058  *   Port_id RTE action structure.
5059  * @param[in] attr
5060  *   Attributes of flow that includes this action.
5061  * @param[out] error
5062  *   Pointer to error structure.
5063  *
5064  * @return
5065  *   0 on success, a negative errno value otherwise and rte_errno is set.
5066  */
5067 static int
5068 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
5069                                 uint64_t action_flags,
5070                                 const struct rte_flow_action *action,
5071                                 const struct rte_flow_attr *attr,
5072                                 struct rte_flow_error *error)
5073 {
5074         const struct rte_flow_action_port_id *port_id;
5075         struct mlx5_priv *act_priv;
5076         struct mlx5_priv *dev_priv;
5077         uint16_t port;
5078
5079         if (!attr->transfer)
5080                 return rte_flow_error_set(error, ENOTSUP,
5081                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5082                                           NULL,
5083                                           "port id action is valid in transfer"
5084                                           " mode only");
5085         if (!action || !action->conf)
5086                 return rte_flow_error_set(error, ENOTSUP,
5087                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5088                                           NULL,
5089                                           "port id action parameters must be"
5090                                           " specified");
5091         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
5092                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5093                 return rte_flow_error_set(error, EINVAL,
5094                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5095                                           "can have only one fate actions in"
5096                                           " a flow");
5097         dev_priv = mlx5_dev_to_eswitch_info(dev);
5098         if (!dev_priv)
5099                 return rte_flow_error_set(error, rte_errno,
5100                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5101                                           NULL,
5102                                           "failed to obtain E-Switch info");
5103         port_id = action->conf;
5104         port = port_id->original ? dev->data->port_id : port_id->id;
5105         act_priv = mlx5_port_to_eswitch_info(port, false);
5106         if (!act_priv)
5107                 return rte_flow_error_set
5108                                 (error, rte_errno,
5109                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
5110                                  "failed to obtain E-Switch port id for port");
5111         if (act_priv->domain_id != dev_priv->domain_id)
5112                 return rte_flow_error_set
5113                                 (error, EINVAL,
5114                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5115                                  "port does not belong to"
5116                                  " E-Switch being configured");
5117         return 0;
5118 }
5119
5120 /**
5121  * Get the maximum number of modify header actions.
5122  *
5123  * @param dev
5124  *   Pointer to rte_eth_dev structure.
5125  * @param root
5126  *   Whether action is on root table.
5127  *
5128  * @return
5129  *   Max number of modify header actions device can support.
5130  */
5131 static inline unsigned int
5132 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
5133                               bool root)
5134 {
5135         /*
5136          * There's no way to directly query the max capacity from FW.
5137          * The maximal value on root table should be assumed to be supported.
5138          */
5139         if (!root)
5140                 return MLX5_MAX_MODIFY_NUM;
5141         else
5142                 return MLX5_ROOT_TBL_MODIFY_NUM;
5143 }
5144
5145 /**
5146  * Validate the meter action.
5147  *
5148  * @param[in] dev
5149  *   Pointer to rte_eth_dev structure.
5150  * @param[in] action_flags
5151  *   Bit-fields that holds the actions detected until now.
5152  * @param[in] action
5153  *   Pointer to the meter action.
5154  * @param[in] attr
5155  *   Attributes of flow that includes this action.
5156  * @param[in] port_id_item
5157  *   Pointer to item indicating port id.
5158  * @param[out] error
5159  *   Pointer to error structure.
5160  *
5161  * @return
5162  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5163  */
5164 static int
5165 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
5166                                 uint64_t action_flags,
5167                                 const struct rte_flow_action *action,
5168                                 const struct rte_flow_attr *attr,
5169                                 const struct rte_flow_item *port_id_item,
5170                                 bool *def_policy,
5171                                 struct rte_flow_error *error)
5172 {
5173         struct mlx5_priv *priv = dev->data->dev_private;
5174         const struct rte_flow_action_meter *am = action->conf;
5175         struct mlx5_flow_meter_info *fm;
5176         struct mlx5_flow_meter_policy *mtr_policy;
5177         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
5178
5179         if (!am)
5180                 return rte_flow_error_set(error, EINVAL,
5181                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5182                                           "meter action conf is NULL");
5183
5184         if (action_flags & MLX5_FLOW_ACTION_METER)
5185                 return rte_flow_error_set(error, ENOTSUP,
5186                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5187                                           "meter chaining not support");
5188         if (action_flags & MLX5_FLOW_ACTION_JUMP)
5189                 return rte_flow_error_set(error, ENOTSUP,
5190                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5191                                           "meter with jump not support");
5192         if (!priv->mtr_en)
5193                 return rte_flow_error_set(error, ENOTSUP,
5194                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5195                                           NULL,
5196                                           "meter action not supported");
5197         fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL);
5198         if (!fm)
5199                 return rte_flow_error_set(error, EINVAL,
5200                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5201                                           "Meter not found");
5202         /* aso meter can always be shared by different domains */
5203         if (fm->ref_cnt && !priv->sh->meter_aso_en &&
5204             !(fm->transfer == attr->transfer ||
5205               (!fm->ingress && !attr->ingress && attr->egress) ||
5206               (!fm->egress && !attr->egress && attr->ingress)))
5207                 return rte_flow_error_set(error, EINVAL,
5208                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5209                         "Flow attributes domain are either invalid "
5210                         "or have a domain conflict with current "
5211                         "meter attributes");
5212         if (fm->def_policy) {
5213                 if (!((attr->transfer &&
5214                         mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) ||
5215                         (attr->egress &&
5216                         mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) ||
5217                         (attr->ingress &&
5218                         mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS])))
5219                         return rte_flow_error_set(error, EINVAL,
5220                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5221                                           "Flow attributes domain "
5222                                           "have a conflict with current "
5223                                           "meter domain attributes");
5224                 *def_policy = true;
5225         } else {
5226                 mtr_policy = mlx5_flow_meter_policy_find(dev,
5227                                                 fm->policy_id, NULL);
5228                 if (!mtr_policy)
5229                         return rte_flow_error_set(error, EINVAL,
5230                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5231                                           "Invalid policy id for meter ");
5232                 if (!((attr->transfer && mtr_policy->transfer) ||
5233                         (attr->egress && mtr_policy->egress) ||
5234                         (attr->ingress && mtr_policy->ingress)))
5235                         return rte_flow_error_set(error, EINVAL,
5236                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5237                                           "Flow attributes domain "
5238                                           "have a conflict with current "
5239                                           "meter domain attributes");
5240                 if (attr->transfer && mtr_policy->dev) {
5241                         /**
5242                          * When policy has fate action of port_id,
5243                          * the flow should have the same src port as policy.
5244                          */
5245                         struct mlx5_priv *policy_port_priv =
5246                                         mtr_policy->dev->data->dev_private;
5247                         int32_t flow_src_port = priv->representor_id;
5248
5249                         if (port_id_item) {
5250                                 const struct rte_flow_item_port_id *spec =
5251                                                         port_id_item->spec;
5252                                 struct mlx5_priv *port_priv =
5253                                         mlx5_port_to_eswitch_info(spec->id,
5254                                                                   false);
5255                                 if (!port_priv)
5256                                         return rte_flow_error_set(error,
5257                                                 rte_errno,
5258                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5259                                                 spec,
5260                                                 "Failed to get port info.");
5261                                 flow_src_port = port_priv->representor_id;
5262                         }
5263                         if (flow_src_port != policy_port_priv->representor_id)
5264                                 return rte_flow_error_set(error,
5265                                                 rte_errno,
5266                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5267                                                 NULL,
5268                                                 "Flow and meter policy "
5269                                                 "have different src port.");
5270                 }
5271                 *def_policy = false;
5272         }
5273         return 0;
5274 }
5275
5276 /**
5277  * Validate the age action.
5278  *
5279  * @param[in] action_flags
5280  *   Holds the actions detected until now.
5281  * @param[in] action
5282  *   Pointer to the age action.
5283  * @param[in] dev
5284  *   Pointer to the Ethernet device structure.
5285  * @param[out] error
5286  *   Pointer to error structure.
5287  *
5288  * @return
5289  *   0 on success, a negative errno value otherwise and rte_errno is set.
5290  */
5291 static int
5292 flow_dv_validate_action_age(uint64_t action_flags,
5293                             const struct rte_flow_action *action,
5294                             struct rte_eth_dev *dev,
5295                             struct rte_flow_error *error)
5296 {
5297         struct mlx5_priv *priv = dev->data->dev_private;
5298         const struct rte_flow_action_age *age = action->conf;
5299
5300         if (!priv->config.devx || (priv->sh->cmng.counter_fallback &&
5301             !priv->sh->aso_age_mng))
5302                 return rte_flow_error_set(error, ENOTSUP,
5303                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5304                                           NULL,
5305                                           "age action not supported");
5306         if (!(action->conf))
5307                 return rte_flow_error_set(error, EINVAL,
5308                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5309                                           "configuration cannot be null");
5310         if (!(age->timeout))
5311                 return rte_flow_error_set(error, EINVAL,
5312                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5313                                           "invalid timeout value 0");
5314         if (action_flags & MLX5_FLOW_ACTION_AGE)
5315                 return rte_flow_error_set(error, EINVAL,
5316                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5317                                           "duplicate age actions set");
5318         return 0;
5319 }
5320
5321 /**
5322  * Validate the modify-header IPv4 DSCP actions.
5323  *
5324  * @param[in] action_flags
5325  *   Holds the actions detected until now.
5326  * @param[in] action
5327  *   Pointer to the modify action.
5328  * @param[in] item_flags
5329  *   Holds the items detected.
5330  * @param[out] error
5331  *   Pointer to error structure.
5332  *
5333  * @return
5334  *   0 on success, a negative errno value otherwise and rte_errno is set.
5335  */
5336 static int
5337 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
5338                                          const struct rte_flow_action *action,
5339                                          const uint64_t item_flags,
5340                                          struct rte_flow_error *error)
5341 {
5342         int ret = 0;
5343
5344         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5345         if (!ret) {
5346                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
5347                         return rte_flow_error_set(error, EINVAL,
5348                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5349                                                   NULL,
5350                                                   "no ipv4 item in pattern");
5351         }
5352         return ret;
5353 }
5354
5355 /**
5356  * Validate the modify-header IPv6 DSCP actions.
5357  *
5358  * @param[in] action_flags
5359  *   Holds the actions detected until now.
5360  * @param[in] action
5361  *   Pointer to the modify action.
5362  * @param[in] item_flags
5363  *   Holds the items detected.
5364  * @param[out] error
5365  *   Pointer to error structure.
5366  *
5367  * @return
5368  *   0 on success, a negative errno value otherwise and rte_errno is set.
5369  */
5370 static int
5371 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
5372                                          const struct rte_flow_action *action,
5373                                          const uint64_t item_flags,
5374                                          struct rte_flow_error *error)
5375 {
5376         int ret = 0;
5377
5378         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5379         if (!ret) {
5380                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
5381                         return rte_flow_error_set(error, EINVAL,
5382                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5383                                                   NULL,
5384                                                   "no ipv6 item in pattern");
5385         }
5386         return ret;
5387 }
5388
5389 int
5390 flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
5391                         struct mlx5_list_entry *entry, void *cb_ctx)
5392 {
5393         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5394         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5395         struct mlx5_flow_dv_modify_hdr_resource *resource =
5396                                   container_of(entry, typeof(*resource), entry);
5397         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5398
5399         key_len += ref->actions_num * sizeof(ref->actions[0]);
5400         return ref->actions_num != resource->actions_num ||
5401                memcmp(&ref->ft_type, &resource->ft_type, key_len);
5402 }
5403
5404 static struct mlx5_indexed_pool *
5405 flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
5406 {
5407         struct mlx5_indexed_pool *ipool = __atomic_load_n
5408                                      (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
5409
5410         if (!ipool) {
5411                 struct mlx5_indexed_pool *expected = NULL;
5412                 struct mlx5_indexed_pool_config cfg =
5413                     (struct mlx5_indexed_pool_config) {
5414                        .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
5415                                                                    (index + 1) *
5416                                            sizeof(struct mlx5_modification_cmd),
5417                        .trunk_size = 64,
5418                        .grow_trunk = 3,
5419                        .grow_shift = 2,
5420                        .need_lock = 1,
5421                        .release_mem_en = !!sh->reclaim_mode,
5422                        .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
5423                        .malloc = mlx5_malloc,
5424                        .free = mlx5_free,
5425                        .type = "mlx5_modify_action_resource",
5426                 };
5427
5428                 cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
5429                 ipool = mlx5_ipool_create(&cfg);
5430                 if (!ipool)
5431                         return NULL;
5432                 if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
5433                                                  &expected, ipool, false,
5434                                                  __ATOMIC_SEQ_CST,
5435                                                  __ATOMIC_SEQ_CST)) {
5436                         mlx5_ipool_destroy(ipool);
5437                         ipool = __atomic_load_n(&sh->mdh_ipools[index],
5438                                                 __ATOMIC_SEQ_CST);
5439                 }
5440         }
5441         return ipool;
5442 }
5443
5444 struct mlx5_list_entry *
5445 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
5446 {
5447         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5448         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5449         struct mlx5dv_dr_domain *ns;
5450         struct mlx5_flow_dv_modify_hdr_resource *entry;
5451         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5452         struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
5453                                                           ref->actions_num - 1);
5454         int ret;
5455         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5456         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5457         uint32_t idx;
5458
5459         if (unlikely(!ipool)) {
5460                 rte_flow_error_set(ctx->error, ENOMEM,
5461                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5462                                    NULL, "cannot allocate modify ipool");
5463                 return NULL;
5464         }
5465         entry = mlx5_ipool_zmalloc(ipool, &idx);
5466         if (!entry) {
5467                 rte_flow_error_set(ctx->error, ENOMEM,
5468                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5469                                    "cannot allocate resource memory");
5470                 return NULL;
5471         }
5472         rte_memcpy(&entry->ft_type,
5473                    RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
5474                    key_len + data_len);
5475         if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
5476                 ns = sh->fdb_domain;
5477         else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
5478                 ns = sh->tx_domain;
5479         else
5480                 ns = sh->rx_domain;
5481         ret = mlx5_flow_os_create_flow_action_modify_header
5482                                         (sh->ctx, ns, entry,
5483                                          data_len, &entry->action);
5484         if (ret) {
5485                 mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
5486                 rte_flow_error_set(ctx->error, ENOMEM,
5487                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5488                                    NULL, "cannot create modification action");
5489                 return NULL;
5490         }
5491         entry->idx = idx;
5492         return &entry->entry;
5493 }
5494
5495 struct mlx5_list_entry *
5496 flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
5497                         void *cb_ctx)
5498 {
5499         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5500         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5501         struct mlx5_flow_dv_modify_hdr_resource *entry;
5502         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5503         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5504         uint32_t idx;
5505
5506         entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
5507                                   &idx);
5508         if (!entry) {
5509                 rte_flow_error_set(ctx->error, ENOMEM,
5510                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5511                                    "cannot allocate resource memory");
5512                 return NULL;
5513         }
5514         memcpy(entry, oentry, sizeof(*entry) + data_len);
5515         entry->idx = idx;
5516         return &entry->entry;
5517 }
5518
5519 void
5520 flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
5521 {
5522         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5523         struct mlx5_flow_dv_modify_hdr_resource *res =
5524                 container_of(entry, typeof(*res), entry);
5525
5526         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
5527 }
5528
5529 /**
5530  * Validate the sample action.
5531  *
5532  * @param[in, out] action_flags
5533  *   Holds the actions detected until now.
5534  * @param[in] action
5535  *   Pointer to the sample action.
5536  * @param[in] dev
5537  *   Pointer to the Ethernet device structure.
5538  * @param[in] attr
5539  *   Attributes of flow that includes this action.
5540  * @param[in] item_flags
5541  *   Holds the items detected.
5542  * @param[in] rss
5543  *   Pointer to the RSS action.
5544  * @param[out] sample_rss
5545  *   Pointer to the RSS action in sample action list.
5546  * @param[out] count
5547  *   Pointer to the COUNT action in sample action list.
5548  * @param[out] fdb_mirror_limit
5549  *   Pointer to the FDB mirror limitation flag.
5550  * @param[out] error
5551  *   Pointer to error structure.
5552  *
5553  * @return
5554  *   0 on success, a negative errno value otherwise and rte_errno is set.
5555  */
5556 static int
5557 flow_dv_validate_action_sample(uint64_t *action_flags,
5558                                const struct rte_flow_action *action,
5559                                struct rte_eth_dev *dev,
5560                                const struct rte_flow_attr *attr,
5561                                uint64_t item_flags,
5562                                const struct rte_flow_action_rss *rss,
5563                                const struct rte_flow_action_rss **sample_rss,
5564                                const struct rte_flow_action_count **count,
5565                                int *fdb_mirror_limit,
5566                                struct rte_flow_error *error)
5567 {
5568         struct mlx5_priv *priv = dev->data->dev_private;
5569         struct mlx5_dev_config *dev_conf = &priv->config;
5570         const struct rte_flow_action_sample *sample = action->conf;
5571         const struct rte_flow_action *act;
5572         uint64_t sub_action_flags = 0;
5573         uint16_t queue_index = 0xFFFF;
5574         int actions_n = 0;
5575         int ret;
5576
5577         if (!sample)
5578                 return rte_flow_error_set(error, EINVAL,
5579                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5580                                           "configuration cannot be NULL");
5581         if (sample->ratio == 0)
5582                 return rte_flow_error_set(error, EINVAL,
5583                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5584                                           "ratio value starts from 1");
5585         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
5586                 return rte_flow_error_set(error, ENOTSUP,
5587                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5588                                           NULL,
5589                                           "sample action not supported");
5590         if (*action_flags & MLX5_FLOW_ACTION_SAMPLE)
5591                 return rte_flow_error_set(error, EINVAL,
5592                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5593                                           "Multiple sample actions not "
5594                                           "supported");
5595         if (*action_flags & MLX5_FLOW_ACTION_METER)
5596                 return rte_flow_error_set(error, EINVAL,
5597                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5598                                           "wrong action order, meter should "
5599                                           "be after sample action");
5600         if (*action_flags & MLX5_FLOW_ACTION_JUMP)
5601                 return rte_flow_error_set(error, EINVAL,
5602                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5603                                           "wrong action order, jump should "
5604                                           "be after sample action");
5605         act = sample->actions;
5606         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
5607                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5608                         return rte_flow_error_set(error, ENOTSUP,
5609                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5610                                                   act, "too many actions");
5611                 switch (act->type) {
5612                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5613                         ret = mlx5_flow_validate_action_queue(act,
5614                                                               sub_action_flags,
5615                                                               dev,
5616                                                               attr, error);
5617                         if (ret < 0)
5618                                 return ret;
5619                         queue_index = ((const struct rte_flow_action_queue *)
5620                                                         (act->conf))->index;
5621                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
5622                         ++actions_n;
5623                         break;
5624                 case RTE_FLOW_ACTION_TYPE_RSS:
5625                         *sample_rss = act->conf;
5626                         ret = mlx5_flow_validate_action_rss(act,
5627                                                             sub_action_flags,
5628                                                             dev, attr,
5629                                                             item_flags,
5630                                                             error);
5631                         if (ret < 0)
5632                                 return ret;
5633                         if (rss && *sample_rss &&
5634                             ((*sample_rss)->level != rss->level ||
5635                             (*sample_rss)->types != rss->types))
5636                                 return rte_flow_error_set(error, ENOTSUP,
5637                                         RTE_FLOW_ERROR_TYPE_ACTION,
5638                                         NULL,
5639                                         "Can't use the different RSS types "
5640                                         "or level in the same flow");
5641                         if (*sample_rss != NULL && (*sample_rss)->queue_num)
5642                                 queue_index = (*sample_rss)->queue[0];
5643                         sub_action_flags |= MLX5_FLOW_ACTION_RSS;
5644                         ++actions_n;
5645                         break;
5646                 case RTE_FLOW_ACTION_TYPE_MARK:
5647                         ret = flow_dv_validate_action_mark(dev, act,
5648                                                            sub_action_flags,
5649                                                            attr, error);
5650                         if (ret < 0)
5651                                 return ret;
5652                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
5653                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
5654                                                 MLX5_FLOW_ACTION_MARK_EXT;
5655                         else
5656                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
5657                         ++actions_n;
5658                         break;
5659                 case RTE_FLOW_ACTION_TYPE_COUNT:
5660                         ret = flow_dv_validate_action_count
5661                                 (dev, is_shared_action_count(act),
5662                                  *action_flags | sub_action_flags,
5663                                  error);
5664                         if (ret < 0)
5665                                 return ret;
5666                         *count = act->conf;
5667                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
5668                         *action_flags |= MLX5_FLOW_ACTION_COUNT;
5669                         ++actions_n;
5670                         break;
5671                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5672                         ret = flow_dv_validate_action_port_id(dev,
5673                                                               sub_action_flags,
5674                                                               act,
5675                                                               attr,
5676                                                               error);
5677                         if (ret)
5678                                 return ret;
5679                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5680                         ++actions_n;
5681                         break;
5682                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5683                         ret = flow_dv_validate_action_raw_encap_decap
5684                                 (dev, NULL, act->conf, attr, &sub_action_flags,
5685                                  &actions_n, action, item_flags, error);
5686                         if (ret < 0)
5687                                 return ret;
5688                         ++actions_n;
5689                         break;
5690                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5691                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5692                         ret = flow_dv_validate_action_l2_encap(dev,
5693                                                                sub_action_flags,
5694                                                                act, attr,
5695                                                                error);
5696                         if (ret < 0)
5697                                 return ret;
5698                         sub_action_flags |= MLX5_FLOW_ACTION_ENCAP;
5699                         ++actions_n;
5700                         break;
5701                 default:
5702                         return rte_flow_error_set(error, ENOTSUP,
5703                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5704                                                   NULL,
5705                                                   "Doesn't support optional "
5706                                                   "action");
5707                 }
5708         }
5709         if (attr->ingress && !attr->transfer) {
5710                 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE |
5711                                           MLX5_FLOW_ACTION_RSS)))
5712                         return rte_flow_error_set(error, EINVAL,
5713                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5714                                                   NULL,
5715                                                   "Ingress must has a dest "
5716                                                   "QUEUE for Sample");
5717         } else if (attr->egress && !attr->transfer) {
5718                 return rte_flow_error_set(error, ENOTSUP,
5719                                           RTE_FLOW_ERROR_TYPE_ACTION,
5720                                           NULL,
5721                                           "Sample Only support Ingress "
5722                                           "or E-Switch");
5723         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
5724                 MLX5_ASSERT(attr->transfer);
5725                 if (sample->ratio > 1)
5726                         return rte_flow_error_set(error, ENOTSUP,
5727                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5728                                                   NULL,
5729                                                   "E-Switch doesn't support "
5730                                                   "any optional action "
5731                                                   "for sampling");
5732                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
5733                         return rte_flow_error_set(error, ENOTSUP,
5734                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5735                                                   NULL,
5736                                                   "unsupported action QUEUE");
5737                 if (sub_action_flags & MLX5_FLOW_ACTION_RSS)
5738                         return rte_flow_error_set(error, ENOTSUP,
5739                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5740                                                   NULL,
5741                                                   "unsupported action QUEUE");
5742                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
5743                         return rte_flow_error_set(error, EINVAL,
5744                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5745                                                   NULL,
5746                                                   "E-Switch must has a dest "
5747                                                   "port for mirroring");
5748                 if (!priv->config.hca_attr.reg_c_preserve &&
5749                      priv->representor_id != UINT16_MAX)
5750                         *fdb_mirror_limit = 1;
5751         }
5752         /* Continue validation for Xcap actions.*/
5753         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
5754             (queue_index == 0xFFFF ||
5755              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5756                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5757                      MLX5_FLOW_XCAP_ACTIONS)
5758                         return rte_flow_error_set(error, ENOTSUP,
5759                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5760                                                   NULL, "encap and decap "
5761                                                   "combination aren't "
5762                                                   "supported");
5763                 if (!attr->transfer && attr->ingress && (sub_action_flags &
5764                                                         MLX5_FLOW_ACTION_ENCAP))
5765                         return rte_flow_error_set(error, ENOTSUP,
5766                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5767                                                   NULL, "encap is not supported"
5768                                                   " for ingress traffic");
5769         }
5770         return 0;
5771 }
5772
5773 /**
5774  * Find existing modify-header resource or create and register a new one.
5775  *
5776  * @param dev[in, out]
5777  *   Pointer to rte_eth_dev structure.
5778  * @param[in, out] resource
5779  *   Pointer to modify-header resource.
5780  * @parm[in, out] dev_flow
5781  *   Pointer to the dev_flow.
5782  * @param[out] error
5783  *   pointer to error structure.
5784  *
5785  * @return
5786  *   0 on success otherwise -errno and errno is set.
5787  */
5788 static int
5789 flow_dv_modify_hdr_resource_register
5790                         (struct rte_eth_dev *dev,
5791                          struct mlx5_flow_dv_modify_hdr_resource *resource,
5792                          struct mlx5_flow *dev_flow,
5793                          struct rte_flow_error *error)
5794 {
5795         struct mlx5_priv *priv = dev->data->dev_private;
5796         struct mlx5_dev_ctx_shared *sh = priv->sh;
5797         uint32_t key_len = sizeof(*resource) -
5798                            offsetof(typeof(*resource), ft_type) +
5799                            resource->actions_num * sizeof(resource->actions[0]);
5800         struct mlx5_list_entry *entry;
5801         struct mlx5_flow_cb_ctx ctx = {
5802                 .error = error,
5803                 .data = resource,
5804         };
5805         struct mlx5_hlist *modify_cmds;
5806         uint64_t key64;
5807
5808         modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
5809                                 "hdr_modify",
5810                                 MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
5811                                 true, false, sh,
5812                                 flow_dv_modify_create_cb,
5813                                 flow_dv_modify_match_cb,
5814                                 flow_dv_modify_remove_cb,
5815                                 flow_dv_modify_clone_cb,
5816                                 flow_dv_modify_clone_free_cb);
5817         if (unlikely(!modify_cmds))
5818                 return -rte_errno;
5819         resource->root = !dev_flow->dv.group;
5820         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
5821                                                                 resource->root))
5822                 return rte_flow_error_set(error, EOVERFLOW,
5823                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5824                                           "too many modify header items");
5825         key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
5826         entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
5827         if (!entry)
5828                 return -rte_errno;
5829         resource = container_of(entry, typeof(*resource), entry);
5830         dev_flow->handle->dvh.modify_hdr = resource;
5831         return 0;
5832 }
5833
5834 /**
5835  * Get DV flow counter by index.
5836  *
5837  * @param[in] dev
5838  *   Pointer to the Ethernet device structure.
5839  * @param[in] idx
5840  *   mlx5 flow counter index in the container.
5841  * @param[out] ppool
5842  *   mlx5 flow counter pool in the container.
5843  *
5844  * @return
5845  *   Pointer to the counter, NULL otherwise.
5846  */
5847 static struct mlx5_flow_counter *
5848 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
5849                            uint32_t idx,
5850                            struct mlx5_flow_counter_pool **ppool)
5851 {
5852         struct mlx5_priv *priv = dev->data->dev_private;
5853         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5854         struct mlx5_flow_counter_pool *pool;
5855
5856         /* Decrease to original index and clear shared bit. */
5857         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
5858         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
5859         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
5860         MLX5_ASSERT(pool);
5861         if (ppool)
5862                 *ppool = pool;
5863         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
5864 }
5865
5866 /**
5867  * Check the devx counter belongs to the pool.
5868  *
5869  * @param[in] pool
5870  *   Pointer to the counter pool.
5871  * @param[in] id
5872  *   The counter devx ID.
5873  *
5874  * @return
5875  *   True if counter belongs to the pool, false otherwise.
5876  */
5877 static bool
5878 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
5879 {
5880         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
5881                    MLX5_COUNTERS_PER_POOL;
5882
5883         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
5884                 return true;
5885         return false;
5886 }
5887
5888 /**
5889  * Get a pool by devx counter ID.
5890  *
5891  * @param[in] cmng
5892  *   Pointer to the counter management.
5893  * @param[in] id
5894  *   The counter devx ID.
5895  *
5896  * @return
5897  *   The counter pool pointer if exists, NULL otherwise,
5898  */
5899 static struct mlx5_flow_counter_pool *
5900 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
5901 {
5902         uint32_t i;
5903         struct mlx5_flow_counter_pool *pool = NULL;
5904
5905         rte_spinlock_lock(&cmng->pool_update_sl);
5906         /* Check last used pool. */
5907         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
5908             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
5909                 pool = cmng->pools[cmng->last_pool_idx];
5910                 goto out;
5911         }
5912         /* ID out of range means no suitable pool in the container. */
5913         if (id > cmng->max_id || id < cmng->min_id)
5914                 goto out;
5915         /*
5916          * Find the pool from the end of the container, since mostly counter
5917          * ID is sequence increasing, and the last pool should be the needed
5918          * one.
5919          */
5920         i = cmng->n_valid;
5921         while (i--) {
5922                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
5923
5924                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
5925                         pool = pool_tmp;
5926                         break;
5927                 }
5928         }
5929 out:
5930         rte_spinlock_unlock(&cmng->pool_update_sl);
5931         return pool;
5932 }
5933
5934 /**
5935  * Resize a counter container.
5936  *
5937  * @param[in] dev
5938  *   Pointer to the Ethernet device structure.
5939  *
5940  * @return
5941  *   0 on success, otherwise negative errno value and rte_errno is set.
5942  */
5943 static int
5944 flow_dv_container_resize(struct rte_eth_dev *dev)
5945 {
5946         struct mlx5_priv *priv = dev->data->dev_private;
5947         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5948         void *old_pools = cmng->pools;
5949         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
5950         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
5951         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
5952
5953         if (!pools) {
5954                 rte_errno = ENOMEM;
5955                 return -ENOMEM;
5956         }
5957         if (old_pools)
5958                 memcpy(pools, old_pools, cmng->n *
5959                                        sizeof(struct mlx5_flow_counter_pool *));
5960         cmng->n = resize;
5961         cmng->pools = pools;
5962         if (old_pools)
5963                 mlx5_free(old_pools);
5964         return 0;
5965 }
5966
5967 /**
5968  * Query a devx flow counter.
5969  *
5970  * @param[in] dev
5971  *   Pointer to the Ethernet device structure.
5972  * @param[in] counter
5973  *   Index to the flow counter.
5974  * @param[out] pkts
5975  *   The statistics value of packets.
5976  * @param[out] bytes
5977  *   The statistics value of bytes.
5978  *
5979  * @return
5980  *   0 on success, otherwise a negative errno value and rte_errno is set.
5981  */
5982 static inline int
5983 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
5984                      uint64_t *bytes)
5985 {
5986         struct mlx5_priv *priv = dev->data->dev_private;
5987         struct mlx5_flow_counter_pool *pool = NULL;
5988         struct mlx5_flow_counter *cnt;
5989         int offset;
5990
5991         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5992         MLX5_ASSERT(pool);
5993         if (priv->sh->cmng.counter_fallback)
5994                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
5995                                         0, pkts, bytes, 0, NULL, NULL, 0);
5996         rte_spinlock_lock(&pool->sl);
5997         if (!pool->raw) {
5998                 *pkts = 0;
5999                 *bytes = 0;
6000         } else {
6001                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
6002                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
6003                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
6004         }
6005         rte_spinlock_unlock(&pool->sl);
6006         return 0;
6007 }
6008
6009 /**
6010  * Create and initialize a new counter pool.
6011  *
6012  * @param[in] dev
6013  *   Pointer to the Ethernet device structure.
6014  * @param[out] dcs
6015  *   The devX counter handle.
6016  * @param[in] age
6017  *   Whether the pool is for counter that was allocated for aging.
6018  * @param[in/out] cont_cur
6019  *   Pointer to the container pointer, it will be update in pool resize.
6020  *
6021  * @return
6022  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
6023  */
6024 static struct mlx5_flow_counter_pool *
6025 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
6026                     uint32_t age)
6027 {
6028         struct mlx5_priv *priv = dev->data->dev_private;
6029         struct mlx5_flow_counter_pool *pool;
6030         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6031         bool fallback = priv->sh->cmng.counter_fallback;
6032         uint32_t size = sizeof(*pool);
6033
6034         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
6035         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
6036         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
6037         if (!pool) {
6038                 rte_errno = ENOMEM;
6039                 return NULL;
6040         }
6041         pool->raw = NULL;
6042         pool->is_aged = !!age;
6043         pool->query_gen = 0;
6044         pool->min_dcs = dcs;
6045         rte_spinlock_init(&pool->sl);
6046         rte_spinlock_init(&pool->csl);
6047         TAILQ_INIT(&pool->counters[0]);
6048         TAILQ_INIT(&pool->counters[1]);
6049         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
6050         rte_spinlock_lock(&cmng->pool_update_sl);
6051         pool->index = cmng->n_valid;
6052         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
6053                 mlx5_free(pool);
6054                 rte_spinlock_unlock(&cmng->pool_update_sl);
6055                 return NULL;
6056         }
6057         cmng->pools[pool->index] = pool;
6058         cmng->n_valid++;
6059         if (unlikely(fallback)) {
6060                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
6061
6062                 if (base < cmng->min_id)
6063                         cmng->min_id = base;
6064                 if (base > cmng->max_id)
6065                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
6066                 cmng->last_pool_idx = pool->index;
6067         }
6068         rte_spinlock_unlock(&cmng->pool_update_sl);
6069         return pool;
6070 }
6071
6072 /**
6073  * Prepare a new counter and/or a new counter pool.
6074  *
6075  * @param[in] dev
6076  *   Pointer to the Ethernet device structure.
6077  * @param[out] cnt_free
6078  *   Where to put the pointer of a new counter.
6079  * @param[in] age
6080  *   Whether the pool is for counter that was allocated for aging.
6081  *
6082  * @return
6083  *   The counter pool pointer and @p cnt_free is set on success,
6084  *   NULL otherwise and rte_errno is set.
6085  */
6086 static struct mlx5_flow_counter_pool *
6087 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
6088                              struct mlx5_flow_counter **cnt_free,
6089                              uint32_t age)
6090 {
6091         struct mlx5_priv *priv = dev->data->dev_private;
6092         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6093         struct mlx5_flow_counter_pool *pool;
6094         struct mlx5_counters tmp_tq;
6095         struct mlx5_devx_obj *dcs = NULL;
6096         struct mlx5_flow_counter *cnt;
6097         enum mlx5_counter_type cnt_type =
6098                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6099         bool fallback = priv->sh->cmng.counter_fallback;
6100         uint32_t i;
6101
6102         if (fallback) {
6103                 /* bulk_bitmap must be 0 for single counter allocation. */
6104                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
6105                 if (!dcs)
6106                         return NULL;
6107                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
6108                 if (!pool) {
6109                         pool = flow_dv_pool_create(dev, dcs, age);
6110                         if (!pool) {
6111                                 mlx5_devx_cmd_destroy(dcs);
6112                                 return NULL;
6113                         }
6114                 }
6115                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
6116                 cnt = MLX5_POOL_GET_CNT(pool, i);
6117                 cnt->pool = pool;
6118                 cnt->dcs_when_free = dcs;
6119                 *cnt_free = cnt;
6120                 return pool;
6121         }
6122         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
6123         if (!dcs) {
6124                 rte_errno = ENODATA;
6125                 return NULL;
6126         }
6127         pool = flow_dv_pool_create(dev, dcs, age);
6128         if (!pool) {
6129                 mlx5_devx_cmd_destroy(dcs);
6130                 return NULL;
6131         }
6132         TAILQ_INIT(&tmp_tq);
6133         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
6134                 cnt = MLX5_POOL_GET_CNT(pool, i);
6135                 cnt->pool = pool;
6136                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
6137         }
6138         rte_spinlock_lock(&cmng->csl[cnt_type]);
6139         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
6140         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6141         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
6142         (*cnt_free)->pool = pool;
6143         return pool;
6144 }
6145
6146 /**
6147  * Allocate a flow counter.
6148  *
6149  * @param[in] dev
6150  *   Pointer to the Ethernet device structure.
6151  * @param[in] age
6152  *   Whether the counter was allocated for aging.
6153  *
6154  * @return
6155  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6156  */
6157 static uint32_t
6158 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
6159 {
6160         struct mlx5_priv *priv = dev->data->dev_private;
6161         struct mlx5_flow_counter_pool *pool = NULL;
6162         struct mlx5_flow_counter *cnt_free = NULL;
6163         bool fallback = priv->sh->cmng.counter_fallback;
6164         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6165         enum mlx5_counter_type cnt_type =
6166                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6167         uint32_t cnt_idx;
6168
6169         if (!priv->config.devx) {
6170                 rte_errno = ENOTSUP;
6171                 return 0;
6172         }
6173         /* Get free counters from container. */
6174         rte_spinlock_lock(&cmng->csl[cnt_type]);
6175         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
6176         if (cnt_free)
6177                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
6178         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6179         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
6180                 goto err;
6181         pool = cnt_free->pool;
6182         if (fallback)
6183                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
6184         /* Create a DV counter action only in the first time usage. */
6185         if (!cnt_free->action) {
6186                 uint16_t offset;
6187                 struct mlx5_devx_obj *dcs;
6188                 int ret;
6189
6190                 if (!fallback) {
6191                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
6192                         dcs = pool->min_dcs;
6193                 } else {
6194                         offset = 0;
6195                         dcs = cnt_free->dcs_when_free;
6196                 }
6197                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
6198                                                             &cnt_free->action);
6199                 if (ret) {
6200                         rte_errno = errno;
6201                         goto err;
6202                 }
6203         }
6204         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
6205                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
6206         /* Update the counter reset values. */
6207         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
6208                                  &cnt_free->bytes))
6209                 goto err;
6210         if (!fallback && !priv->sh->cmng.query_thread_on)
6211                 /* Start the asynchronous batch query by the host thread. */
6212                 mlx5_set_query_alarm(priv->sh);
6213         /*
6214          * When the count action isn't shared (by ID), shared_info field is
6215          * used for indirect action API's refcnt.
6216          * When the counter action is not shared neither by ID nor by indirect
6217          * action API, shared info must be 1.
6218          */
6219         cnt_free->shared_info.refcnt = 1;
6220         return cnt_idx;
6221 err:
6222         if (cnt_free) {
6223                 cnt_free->pool = pool;
6224                 if (fallback)
6225                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
6226                 rte_spinlock_lock(&cmng->csl[cnt_type]);
6227                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
6228                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
6229         }
6230         return 0;
6231 }
6232
6233 /**
6234  * Allocate a shared flow counter.
6235  *
6236  * @param[in] ctx
6237  *   Pointer to the shared counter configuration.
6238  * @param[in] data
6239  *   Pointer to save the allocated counter index.
6240  *
6241  * @return
6242  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6243  */
6244
6245 static int32_t
6246 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
6247 {
6248         struct mlx5_shared_counter_conf *conf = ctx;
6249         struct rte_eth_dev *dev = conf->dev;
6250         struct mlx5_flow_counter *cnt;
6251
6252         data->dword = flow_dv_counter_alloc(dev, 0);
6253         data->dword |= MLX5_CNT_SHARED_OFFSET;
6254         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
6255         cnt->shared_info.id = conf->id;
6256         return 0;
6257 }
6258
6259 /**
6260  * Get a shared flow counter.
6261  *
6262  * @param[in] dev
6263  *   Pointer to the Ethernet device structure.
6264  * @param[in] id
6265  *   Counter identifier.
6266  *
6267  * @return
6268  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6269  */
6270 static uint32_t
6271 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
6272 {
6273         struct mlx5_priv *priv = dev->data->dev_private;
6274         struct mlx5_shared_counter_conf conf = {
6275                 .dev = dev,
6276                 .id = id,
6277         };
6278         union mlx5_l3t_data data = {
6279                 .dword = 0,
6280         };
6281
6282         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
6283                                flow_dv_counter_alloc_shared_cb, &conf);
6284         return data.dword;
6285 }
6286
6287 /**
6288  * Get age param from counter index.
6289  *
6290  * @param[in] dev
6291  *   Pointer to the Ethernet device structure.
6292  * @param[in] counter
6293  *   Index to the counter handler.
6294  *
6295  * @return
6296  *   The aging parameter specified for the counter index.
6297  */
6298 static struct mlx5_age_param*
6299 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
6300                                 uint32_t counter)
6301 {
6302         struct mlx5_flow_counter *cnt;
6303         struct mlx5_flow_counter_pool *pool = NULL;
6304
6305         flow_dv_counter_get_by_idx(dev, counter, &pool);
6306         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
6307         cnt = MLX5_POOL_GET_CNT(pool, counter);
6308         return MLX5_CNT_TO_AGE(cnt);
6309 }
6310
6311 /**
6312  * Remove a flow counter from aged counter list.
6313  *
6314  * @param[in] dev
6315  *   Pointer to the Ethernet device structure.
6316  * @param[in] counter
6317  *   Index to the counter handler.
6318  * @param[in] cnt
6319  *   Pointer to the counter handler.
6320  */
6321 static void
6322 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
6323                                 uint32_t counter, struct mlx5_flow_counter *cnt)
6324 {
6325         struct mlx5_age_info *age_info;
6326         struct mlx5_age_param *age_param;
6327         struct mlx5_priv *priv = dev->data->dev_private;
6328         uint16_t expected = AGE_CANDIDATE;
6329
6330         age_info = GET_PORT_AGE_INFO(priv);
6331         age_param = flow_dv_counter_idx_get_age(dev, counter);
6332         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
6333                                          AGE_FREE, false, __ATOMIC_RELAXED,
6334                                          __ATOMIC_RELAXED)) {
6335                 /**
6336                  * We need the lock even it is age timeout,
6337                  * since counter may still in process.
6338                  */
6339                 rte_spinlock_lock(&age_info->aged_sl);
6340                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
6341                 rte_spinlock_unlock(&age_info->aged_sl);
6342                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
6343         }
6344 }
6345
6346 /**
6347  * Release a flow counter.
6348  *
6349  * @param[in] dev
6350  *   Pointer to the Ethernet device structure.
6351  * @param[in] counter
6352  *   Index to the counter handler.
6353  */
6354 static void
6355 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter)
6356 {
6357         struct mlx5_priv *priv = dev->data->dev_private;
6358         struct mlx5_flow_counter_pool *pool = NULL;
6359         struct mlx5_flow_counter *cnt;
6360         enum mlx5_counter_type cnt_type;
6361
6362         if (!counter)
6363                 return;
6364         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6365         MLX5_ASSERT(pool);
6366         if (pool->is_aged) {
6367                 flow_dv_counter_remove_from_age(dev, counter, cnt);
6368         } else {
6369                 /*
6370                  * If the counter action is shared by ID, the l3t_clear_entry
6371                  * function reduces its references counter. If after the
6372                  * reduction the action is still referenced, the function
6373                  * returns here and does not release it.
6374                  */
6375                 if (IS_LEGACY_SHARED_CNT(counter) &&
6376                     mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
6377                                          cnt->shared_info.id))
6378                         return;
6379                 /*
6380                  * If the counter action is shared by indirect action API,
6381                  * the atomic function reduces its references counter.
6382                  * If after the reduction the action is still referenced, the
6383                  * function returns here and does not release it.
6384                  * When the counter action is not shared neither by ID nor by
6385                  * indirect action API, shared info is 1 before the reduction,
6386                  * so this condition is failed and function doesn't return here.
6387                  */
6388                 if (!IS_LEGACY_SHARED_CNT(counter) &&
6389                     __atomic_sub_fetch(&cnt->shared_info.refcnt, 1,
6390                                        __ATOMIC_RELAXED))
6391                         return;
6392         }
6393         cnt->pool = pool;
6394         /*
6395          * Put the counter back to list to be updated in none fallback mode.
6396          * Currently, we are using two list alternately, while one is in query,
6397          * add the freed counter to the other list based on the pool query_gen
6398          * value. After query finishes, add counter the list to the global
6399          * container counter list. The list changes while query starts. In
6400          * this case, lock will not be needed as query callback and release
6401          * function both operate with the different list.
6402          */
6403         if (!priv->sh->cmng.counter_fallback) {
6404                 rte_spinlock_lock(&pool->csl);
6405                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
6406                 rte_spinlock_unlock(&pool->csl);
6407         } else {
6408                 cnt->dcs_when_free = cnt->dcs_when_active;
6409                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6410                                            MLX5_COUNTER_TYPE_ORIGIN;
6411                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
6412                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
6413                                   cnt, next);
6414                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
6415         }
6416 }
6417
6418 /**
6419  * Resize a meter id container.
6420  *
6421  * @param[in] dev
6422  *   Pointer to the Ethernet device structure.
6423  *
6424  * @return
6425  *   0 on success, otherwise negative errno value and rte_errno is set.
6426  */
6427 static int
6428 flow_dv_mtr_container_resize(struct rte_eth_dev *dev)
6429 {
6430         struct mlx5_priv *priv = dev->data->dev_private;
6431         struct mlx5_aso_mtr_pools_mng *pools_mng =
6432                                 &priv->sh->mtrmng->pools_mng;
6433         void *old_pools = pools_mng->pools;
6434         uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE;
6435         uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize;
6436         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
6437
6438         if (!pools) {
6439                 rte_errno = ENOMEM;
6440                 return -ENOMEM;
6441         }
6442         if (!pools_mng->n)
6443                 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) {
6444                         mlx5_free(pools);
6445                         return -ENOMEM;
6446                 }
6447         if (old_pools)
6448                 memcpy(pools, old_pools, pools_mng->n *
6449                                        sizeof(struct mlx5_aso_mtr_pool *));
6450         pools_mng->n = resize;
6451         pools_mng->pools = pools;
6452         if (old_pools)
6453                 mlx5_free(old_pools);
6454         return 0;
6455 }
6456
6457 /**
6458  * Prepare a new meter and/or a new meter pool.
6459  *
6460  * @param[in] dev
6461  *   Pointer to the Ethernet device structure.
6462  * @param[out] mtr_free
6463  *   Where to put the pointer of a new meter.g.
6464  *
6465  * @return
6466  *   The meter pool pointer and @mtr_free is set on success,
6467  *   NULL otherwise and rte_errno is set.
6468  */
6469 static struct mlx5_aso_mtr_pool *
6470 flow_dv_mtr_pool_create(struct rte_eth_dev *dev,
6471                              struct mlx5_aso_mtr **mtr_free)
6472 {
6473         struct mlx5_priv *priv = dev->data->dev_private;
6474         struct mlx5_aso_mtr_pools_mng *pools_mng =
6475                                 &priv->sh->mtrmng->pools_mng;
6476         struct mlx5_aso_mtr_pool *pool = NULL;
6477         struct mlx5_devx_obj *dcs = NULL;
6478         uint32_t i;
6479         uint32_t log_obj_size;
6480
6481         log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
6482         dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx,
6483                         priv->sh->pdn, log_obj_size);
6484         if (!dcs) {
6485                 rte_errno = ENODATA;
6486                 return NULL;
6487         }
6488         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
6489         if (!pool) {
6490                 rte_errno = ENOMEM;
6491                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6492                 return NULL;
6493         }
6494         pool->devx_obj = dcs;
6495         pool->index = pools_mng->n_valid;
6496         if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) {
6497                 mlx5_free(pool);
6498                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6499                 return NULL;
6500         }
6501         pools_mng->pools[pool->index] = pool;
6502         pools_mng->n_valid++;
6503         for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) {
6504                 pool->mtrs[i].offset = i;
6505                 LIST_INSERT_HEAD(&pools_mng->meters,
6506                                                 &pool->mtrs[i], next);
6507         }
6508         pool->mtrs[0].offset = 0;
6509         *mtr_free = &pool->mtrs[0];
6510         return pool;
6511 }
6512
6513 /**
6514  * Release a flow meter into pool.
6515  *
6516  * @param[in] dev
6517  *   Pointer to the Ethernet device structure.
6518  * @param[in] mtr_idx
6519  *   Index to aso flow meter.
6520  */
6521 static void
6522 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx)
6523 {
6524         struct mlx5_priv *priv = dev->data->dev_private;
6525         struct mlx5_aso_mtr_pools_mng *pools_mng =
6526                                 &priv->sh->mtrmng->pools_mng;
6527         struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
6528
6529         MLX5_ASSERT(aso_mtr);
6530         rte_spinlock_lock(&pools_mng->mtrsl);
6531         memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info));
6532         aso_mtr->state = ASO_METER_FREE;
6533         LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next);
6534         rte_spinlock_unlock(&pools_mng->mtrsl);
6535 }
6536
6537 /**
6538  * Allocate a aso flow meter.
6539  *
6540  * @param[in] dev
6541  *   Pointer to the Ethernet device structure.
6542  *
6543  * @return
6544  *   Index to aso flow meter on success, 0 otherwise and rte_errno is set.
6545  */
6546 static uint32_t
6547 flow_dv_mtr_alloc(struct rte_eth_dev *dev)
6548 {
6549         struct mlx5_priv *priv = dev->data->dev_private;
6550         struct mlx5_aso_mtr *mtr_free = NULL;
6551         struct mlx5_aso_mtr_pools_mng *pools_mng =
6552                                 &priv->sh->mtrmng->pools_mng;
6553         struct mlx5_aso_mtr_pool *pool;
6554         uint32_t mtr_idx = 0;
6555
6556         if (!priv->config.devx) {
6557                 rte_errno = ENOTSUP;
6558                 return 0;
6559         }
6560         /* Allocate the flow meter memory. */
6561         /* Get free meters from management. */
6562         rte_spinlock_lock(&pools_mng->mtrsl);
6563         mtr_free = LIST_FIRST(&pools_mng->meters);
6564         if (mtr_free)
6565                 LIST_REMOVE(mtr_free, next);
6566         if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) {
6567                 rte_spinlock_unlock(&pools_mng->mtrsl);
6568                 return 0;
6569         }
6570         mtr_free->state = ASO_METER_WAIT;
6571         rte_spinlock_unlock(&pools_mng->mtrsl);
6572         pool = container_of(mtr_free,
6573                         struct mlx5_aso_mtr_pool,
6574                         mtrs[mtr_free->offset]);
6575         mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
6576         if (!mtr_free->fm.meter_action) {
6577 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
6578                 struct rte_flow_error error;
6579                 uint8_t reg_id;
6580
6581                 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
6582                 mtr_free->fm.meter_action =
6583                         mlx5_glue->dv_create_flow_action_aso
6584                                                 (priv->sh->rx_domain,
6585                                                  pool->devx_obj->obj,
6586                                                  mtr_free->offset,
6587                                                  (1 << MLX5_FLOW_COLOR_GREEN),
6588                                                  reg_id - REG_C_0);
6589 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
6590                 if (!mtr_free->fm.meter_action) {
6591                         flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
6592                         return 0;
6593                 }
6594         }
6595         return mtr_idx;
6596 }
6597
6598 /**
6599  * Verify the @p attributes will be correctly understood by the NIC and store
6600  * them in the @p flow if everything is correct.
6601  *
6602  * @param[in] dev
6603  *   Pointer to dev struct.
6604  * @param[in] attributes
6605  *   Pointer to flow attributes
6606  * @param[in] external
6607  *   This flow rule is created by request external to PMD.
6608  * @param[out] error
6609  *   Pointer to error structure.
6610  *
6611  * @return
6612  *   - 0 on success and non root table.
6613  *   - 1 on success and root table.
6614  *   - a negative errno value otherwise and rte_errno is set.
6615  */
6616 static int
6617 flow_dv_validate_attributes(struct rte_eth_dev *dev,
6618                             const struct mlx5_flow_tunnel *tunnel,
6619                             const struct rte_flow_attr *attributes,
6620                             const struct flow_grp_info *grp_info,
6621                             struct rte_flow_error *error)
6622 {
6623         struct mlx5_priv *priv = dev->data->dev_private;
6624         uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes);
6625         int ret = 0;
6626
6627 #ifndef HAVE_MLX5DV_DR
6628         RTE_SET_USED(tunnel);
6629         RTE_SET_USED(grp_info);
6630         if (attributes->group)
6631                 return rte_flow_error_set(error, ENOTSUP,
6632                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6633                                           NULL,
6634                                           "groups are not supported");
6635 #else
6636         uint32_t table = 0;
6637
6638         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
6639                                        grp_info, error);
6640         if (ret)
6641                 return ret;
6642         if (!table)
6643                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
6644 #endif
6645         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
6646             attributes->priority > lowest_priority)
6647                 return rte_flow_error_set(error, ENOTSUP,
6648                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
6649                                           NULL,
6650                                           "priority out of range");
6651         if (attributes->transfer) {
6652                 if (!priv->config.dv_esw_en)
6653                         return rte_flow_error_set
6654                                 (error, ENOTSUP,
6655                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6656                                  "E-Switch dr is not supported");
6657                 if (!(priv->representor || priv->master))
6658                         return rte_flow_error_set
6659                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6660                                  NULL, "E-Switch configuration can only be"
6661                                  " done by a master or a representor device");
6662                 if (attributes->egress)
6663                         return rte_flow_error_set
6664                                 (error, ENOTSUP,
6665                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
6666                                  "egress is not supported");
6667         }
6668         if (!(attributes->egress ^ attributes->ingress))
6669                 return rte_flow_error_set(error, ENOTSUP,
6670                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
6671                                           "must specify exactly one of "
6672                                           "ingress or egress");
6673         return ret;
6674 }
6675
6676 static uint16_t
6677 mlx5_flow_locate_proto_l3(const struct rte_flow_item **head,
6678                           const struct rte_flow_item *end)
6679 {
6680         const struct rte_flow_item *item = *head;
6681         uint16_t l3_protocol;
6682
6683         for (; item != end; item++) {
6684                 switch (item->type) {
6685                 default:
6686                         break;
6687                 case RTE_FLOW_ITEM_TYPE_IPV4:
6688                         l3_protocol = RTE_ETHER_TYPE_IPV4;
6689                         goto l3_ok;
6690                 case RTE_FLOW_ITEM_TYPE_IPV6:
6691                         l3_protocol = RTE_ETHER_TYPE_IPV6;
6692                         goto l3_ok;
6693                 case RTE_FLOW_ITEM_TYPE_ETH:
6694                         if (item->mask && item->spec) {
6695                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_eth,
6696                                                             type, item,
6697                                                             l3_protocol);
6698                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6699                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6700                                         goto l3_ok;
6701                         }
6702                         break;
6703                 case RTE_FLOW_ITEM_TYPE_VLAN:
6704                         if (item->mask && item->spec) {
6705                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_vlan,
6706                                                             inner_type, item,
6707                                                             l3_protocol);
6708                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6709                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6710                                         goto l3_ok;
6711                         }
6712                         break;
6713                 }
6714         }
6715         return 0;
6716 l3_ok:
6717         *head = item;
6718         return l3_protocol;
6719 }
6720
6721 static uint8_t
6722 mlx5_flow_locate_proto_l4(const struct rte_flow_item **head,
6723                           const struct rte_flow_item *end)
6724 {
6725         const struct rte_flow_item *item = *head;
6726         uint8_t l4_protocol;
6727
6728         for (; item != end; item++) {
6729                 switch (item->type) {
6730                 default:
6731                         break;
6732                 case RTE_FLOW_ITEM_TYPE_TCP:
6733                         l4_protocol = IPPROTO_TCP;
6734                         goto l4_ok;
6735                 case RTE_FLOW_ITEM_TYPE_UDP:
6736                         l4_protocol = IPPROTO_UDP;
6737                         goto l4_ok;
6738                 case RTE_FLOW_ITEM_TYPE_IPV4:
6739                         if (item->mask && item->spec) {
6740                                 const struct rte_flow_item_ipv4 *mask, *spec;
6741
6742                                 mask = (typeof(mask))item->mask;
6743                                 spec = (typeof(spec))item->spec;
6744                                 l4_protocol = mask->hdr.next_proto_id &
6745                                               spec->hdr.next_proto_id;
6746                                 if (l4_protocol == IPPROTO_TCP ||
6747                                     l4_protocol == IPPROTO_UDP)
6748                                         goto l4_ok;
6749                         }
6750                         break;
6751                 case RTE_FLOW_ITEM_TYPE_IPV6:
6752                         if (item->mask && item->spec) {
6753                                 const struct rte_flow_item_ipv6 *mask, *spec;
6754                                 mask = (typeof(mask))item->mask;
6755                                 spec = (typeof(spec))item->spec;
6756                                 l4_protocol = mask->hdr.proto & spec->hdr.proto;
6757                                 if (l4_protocol == IPPROTO_TCP ||
6758                                     l4_protocol == IPPROTO_UDP)
6759                                         goto l4_ok;
6760                         }
6761                         break;
6762                 }
6763         }
6764         return 0;
6765 l4_ok:
6766         *head = item;
6767         return l4_protocol;
6768 }
6769
6770 static int
6771 flow_dv_validate_item_integrity(struct rte_eth_dev *dev,
6772                                 const struct rte_flow_item *rule_items,
6773                                 const struct rte_flow_item *integrity_item,
6774                                 struct rte_flow_error *error)
6775 {
6776         struct mlx5_priv *priv = dev->data->dev_private;
6777         const struct rte_flow_item *tunnel_item, *end_item, *item = rule_items;
6778         const struct rte_flow_item_integrity *mask = (typeof(mask))
6779                                                      integrity_item->mask;
6780         const struct rte_flow_item_integrity *spec = (typeof(spec))
6781                                                      integrity_item->spec;
6782         uint32_t protocol;
6783
6784         if (!priv->config.hca_attr.pkt_integrity_match)
6785                 return rte_flow_error_set(error, ENOTSUP,
6786                                           RTE_FLOW_ERROR_TYPE_ITEM,
6787                                           integrity_item,
6788                                           "packet integrity integrity_item not supported");
6789         if (!mask)
6790                 mask = &rte_flow_item_integrity_mask;
6791         if (!mlx5_validate_integrity_item(mask))
6792                 return rte_flow_error_set(error, ENOTSUP,
6793                                           RTE_FLOW_ERROR_TYPE_ITEM,
6794                                           integrity_item,
6795                                           "unsupported integrity filter");
6796         tunnel_item = mlx5_flow_find_tunnel_item(rule_items);
6797         if (spec->level > 1) {
6798                 if (!tunnel_item)
6799                         return rte_flow_error_set(error, ENOTSUP,
6800                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6801                                                   integrity_item,
6802                                                   "missing tunnel item");
6803                 item = tunnel_item;
6804                 end_item = mlx5_find_end_item(tunnel_item);
6805         } else {
6806                 end_item = tunnel_item ? tunnel_item :
6807                            mlx5_find_end_item(integrity_item);
6808         }
6809         if (mask->l3_ok || mask->ipv4_csum_ok) {
6810                 protocol = mlx5_flow_locate_proto_l3(&item, end_item);
6811                 if (!protocol)
6812                         return rte_flow_error_set(error, EINVAL,
6813                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6814                                                   integrity_item,
6815                                                   "missing L3 protocol");
6816         }
6817         if (mask->l4_ok || mask->l4_csum_ok) {
6818                 protocol = mlx5_flow_locate_proto_l4(&item, end_item);
6819                 if (!protocol)
6820                         return rte_flow_error_set(error, EINVAL,
6821                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6822                                                   integrity_item,
6823                                                   "missing L4 protocol");
6824         }
6825         return 0;
6826 }
6827
6828 /**
6829  * Internal validation function. For validating both actions and items.
6830  *
6831  * @param[in] dev
6832  *   Pointer to the rte_eth_dev structure.
6833  * @param[in] attr
6834  *   Pointer to the flow attributes.
6835  * @param[in] items
6836  *   Pointer to the list of items.
6837  * @param[in] actions
6838  *   Pointer to the list of actions.
6839  * @param[in] external
6840  *   This flow rule is created by request external to PMD.
6841  * @param[in] hairpin
6842  *   Number of hairpin TX actions, 0 means classic flow.
6843  * @param[out] error
6844  *   Pointer to the error structure.
6845  *
6846  * @return
6847  *   0 on success, a negative errno value otherwise and rte_errno is set.
6848  */
6849 static int
6850 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
6851                  const struct rte_flow_item items[],
6852                  const struct rte_flow_action actions[],
6853                  bool external, int hairpin, struct rte_flow_error *error)
6854 {
6855         int ret;
6856         uint64_t action_flags = 0;
6857         uint64_t item_flags = 0;
6858         uint64_t last_item = 0;
6859         uint8_t next_protocol = 0xff;
6860         uint16_t ether_type = 0;
6861         int actions_n = 0;
6862         uint8_t item_ipv6_proto = 0;
6863         int fdb_mirror_limit = 0;
6864         int modify_after_mirror = 0;
6865         const struct rte_flow_item *geneve_item = NULL;
6866         const struct rte_flow_item *gre_item = NULL;
6867         const struct rte_flow_item *gtp_item = NULL;
6868         const struct rte_flow_action_raw_decap *decap;
6869         const struct rte_flow_action_raw_encap *encap;
6870         const struct rte_flow_action_rss *rss = NULL;
6871         const struct rte_flow_action_rss *sample_rss = NULL;
6872         const struct rte_flow_action_count *sample_count = NULL;
6873         const struct rte_flow_item_tcp nic_tcp_mask = {
6874                 .hdr = {
6875                         .tcp_flags = 0xFF,
6876                         .src_port = RTE_BE16(UINT16_MAX),
6877                         .dst_port = RTE_BE16(UINT16_MAX),
6878                 }
6879         };
6880         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
6881                 .hdr = {
6882                         .src_addr =
6883                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6884                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6885                         .dst_addr =
6886                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6887                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6888                         .vtc_flow = RTE_BE32(0xffffffff),
6889                         .proto = 0xff,
6890                         .hop_limits = 0xff,
6891                 },
6892                 .has_frag_ext = 1,
6893         };
6894         const struct rte_flow_item_ecpri nic_ecpri_mask = {
6895                 .hdr = {
6896                         .common = {
6897                                 .u32 =
6898                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
6899                                         .type = 0xFF,
6900                                         }).u32),
6901                         },
6902                         .dummy[0] = 0xffffffff,
6903                 },
6904         };
6905         struct mlx5_priv *priv = dev->data->dev_private;
6906         struct mlx5_dev_config *dev_conf = &priv->config;
6907         uint16_t queue_index = 0xFFFF;
6908         const struct rte_flow_item_vlan *vlan_m = NULL;
6909         uint32_t rw_act_num = 0;
6910         uint64_t is_root;
6911         const struct mlx5_flow_tunnel *tunnel;
6912         enum mlx5_tof_rule_type tof_rule_type;
6913         struct flow_grp_info grp_info = {
6914                 .external = !!external,
6915                 .transfer = !!attr->transfer,
6916                 .fdb_def_rule = !!priv->fdb_def_rule,
6917                 .std_tbl_fix = true,
6918         };
6919         const struct rte_eth_hairpin_conf *conf;
6920         const struct rte_flow_item *rule_items = items;
6921         const struct rte_flow_item *port_id_item = NULL;
6922         bool def_policy = false;
6923         uint16_t udp_dport = 0;
6924
6925         if (items == NULL)
6926                 return -1;
6927         tunnel = is_tunnel_offload_active(dev) ?
6928                  mlx5_get_tof(items, actions, &tof_rule_type) : NULL;
6929         if (tunnel) {
6930                 if (priv->representor)
6931                         return rte_flow_error_set
6932                                 (error, ENOTSUP,
6933                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6934                                  NULL, "decap not supported for VF representor");
6935                 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE)
6936                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6937                 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE)
6938                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
6939                                         MLX5_FLOW_ACTION_DECAP;
6940                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
6941                                         (dev, attr, tunnel, tof_rule_type);
6942         }
6943         ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error);
6944         if (ret < 0)
6945                 return ret;
6946         is_root = (uint64_t)ret;
6947         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6948                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6949                 int type = items->type;
6950
6951                 if (!mlx5_flow_os_item_supported(type))
6952                         return rte_flow_error_set(error, ENOTSUP,
6953                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6954                                                   NULL, "item not supported");
6955                 switch (type) {
6956                 case RTE_FLOW_ITEM_TYPE_VOID:
6957                         break;
6958                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6959                         ret = flow_dv_validate_item_port_id
6960                                         (dev, items, attr, item_flags, error);
6961                         if (ret < 0)
6962                                 return ret;
6963                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6964                         port_id_item = items;
6965                         break;
6966                 case RTE_FLOW_ITEM_TYPE_ETH:
6967                         ret = mlx5_flow_validate_item_eth(items, item_flags,
6968                                                           true, error);
6969                         if (ret < 0)
6970                                 return ret;
6971                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
6972                                              MLX5_FLOW_LAYER_OUTER_L2;
6973                         if (items->mask != NULL && items->spec != NULL) {
6974                                 ether_type =
6975                                         ((const struct rte_flow_item_eth *)
6976                                          items->spec)->type;
6977                                 ether_type &=
6978                                         ((const struct rte_flow_item_eth *)
6979                                          items->mask)->type;
6980                                 ether_type = rte_be_to_cpu_16(ether_type);
6981                         } else {
6982                                 ether_type = 0;
6983                         }
6984                         break;
6985                 case RTE_FLOW_ITEM_TYPE_VLAN:
6986                         ret = flow_dv_validate_item_vlan(items, item_flags,
6987                                                          dev, error);
6988                         if (ret < 0)
6989                                 return ret;
6990                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
6991                                              MLX5_FLOW_LAYER_OUTER_VLAN;
6992                         if (items->mask != NULL && items->spec != NULL) {
6993                                 ether_type =
6994                                         ((const struct rte_flow_item_vlan *)
6995                                          items->spec)->inner_type;
6996                                 ether_type &=
6997                                         ((const struct rte_flow_item_vlan *)
6998                                          items->mask)->inner_type;
6999                                 ether_type = rte_be_to_cpu_16(ether_type);
7000                         } else {
7001                                 ether_type = 0;
7002                         }
7003                         /* Store outer VLAN mask for of_push_vlan action. */
7004                         if (!tunnel)
7005                                 vlan_m = items->mask;
7006                         break;
7007                 case RTE_FLOW_ITEM_TYPE_IPV4:
7008                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7009                                                   &item_flags, &tunnel);
7010                         ret = flow_dv_validate_item_ipv4(dev, items, item_flags,
7011                                                          last_item, ether_type,
7012                                                          error);
7013                         if (ret < 0)
7014                                 return ret;
7015                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
7016                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
7017                         if (items->mask != NULL &&
7018                             ((const struct rte_flow_item_ipv4 *)
7019                              items->mask)->hdr.next_proto_id) {
7020                                 next_protocol =
7021                                         ((const struct rte_flow_item_ipv4 *)
7022                                          (items->spec))->hdr.next_proto_id;
7023                                 next_protocol &=
7024                                         ((const struct rte_flow_item_ipv4 *)
7025                                          (items->mask))->hdr.next_proto_id;
7026                         } else {
7027                                 /* Reset for inner layer. */
7028                                 next_protocol = 0xff;
7029                         }
7030                         break;
7031                 case RTE_FLOW_ITEM_TYPE_IPV6:
7032                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7033                                                   &item_flags, &tunnel);
7034                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
7035                                                            last_item,
7036                                                            ether_type,
7037                                                            &nic_ipv6_mask,
7038                                                            error);
7039                         if (ret < 0)
7040                                 return ret;
7041                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
7042                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
7043                         if (items->mask != NULL &&
7044                             ((const struct rte_flow_item_ipv6 *)
7045                              items->mask)->hdr.proto) {
7046                                 item_ipv6_proto =
7047                                         ((const struct rte_flow_item_ipv6 *)
7048                                          items->spec)->hdr.proto;
7049                                 next_protocol =
7050                                         ((const struct rte_flow_item_ipv6 *)
7051                                          items->spec)->hdr.proto;
7052                                 next_protocol &=
7053                                         ((const struct rte_flow_item_ipv6 *)
7054                                          items->mask)->hdr.proto;
7055                         } else {
7056                                 /* Reset for inner layer. */
7057                                 next_protocol = 0xff;
7058                         }
7059                         break;
7060                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
7061                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
7062                                                                   item_flags,
7063                                                                   error);
7064                         if (ret < 0)
7065                                 return ret;
7066                         last_item = tunnel ?
7067                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
7068                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
7069                         if (items->mask != NULL &&
7070                             ((const struct rte_flow_item_ipv6_frag_ext *)
7071                              items->mask)->hdr.next_header) {
7072                                 next_protocol =
7073                                 ((const struct rte_flow_item_ipv6_frag_ext *)
7074                                  items->spec)->hdr.next_header;
7075                                 next_protocol &=
7076                                 ((const struct rte_flow_item_ipv6_frag_ext *)
7077                                  items->mask)->hdr.next_header;
7078                         } else {
7079                                 /* Reset for inner layer. */
7080                                 next_protocol = 0xff;
7081                         }
7082                         break;
7083                 case RTE_FLOW_ITEM_TYPE_TCP:
7084                         ret = mlx5_flow_validate_item_tcp
7085                                                 (items, item_flags,
7086                                                  next_protocol,
7087                                                  &nic_tcp_mask,
7088                                                  error);
7089                         if (ret < 0)
7090                                 return ret;
7091                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
7092                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
7093                         break;
7094                 case RTE_FLOW_ITEM_TYPE_UDP:
7095                         ret = mlx5_flow_validate_item_udp(items, item_flags,
7096                                                           next_protocol,
7097                                                           error);
7098                         const struct rte_flow_item_udp *spec = items->spec;
7099                         const struct rte_flow_item_udp *mask = items->mask;
7100                         if (!mask)
7101                                 mask = &rte_flow_item_udp_mask;
7102                         if (spec != NULL)
7103                                 udp_dport = rte_be_to_cpu_16
7104                                                 (spec->hdr.dst_port &
7105                                                  mask->hdr.dst_port);
7106                         if (ret < 0)
7107                                 return ret;
7108                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
7109                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
7110                         break;
7111                 case RTE_FLOW_ITEM_TYPE_GRE:
7112                         ret = mlx5_flow_validate_item_gre(items, item_flags,
7113                                                           next_protocol, error);
7114                         if (ret < 0)
7115                                 return ret;
7116                         gre_item = items;
7117                         last_item = MLX5_FLOW_LAYER_GRE;
7118                         break;
7119                 case RTE_FLOW_ITEM_TYPE_NVGRE:
7120                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
7121                                                             next_protocol,
7122                                                             error);
7123                         if (ret < 0)
7124                                 return ret;
7125                         last_item = MLX5_FLOW_LAYER_NVGRE;
7126                         break;
7127                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
7128                         ret = mlx5_flow_validate_item_gre_key
7129                                 (items, item_flags, gre_item, error);
7130                         if (ret < 0)
7131                                 return ret;
7132                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
7133                         break;
7134                 case RTE_FLOW_ITEM_TYPE_VXLAN:
7135                         ret = mlx5_flow_validate_item_vxlan(dev, udp_dport,
7136                                                             items, item_flags,
7137                                                             attr, error);
7138                         if (ret < 0)
7139                                 return ret;
7140                         last_item = MLX5_FLOW_LAYER_VXLAN;
7141                         break;
7142                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
7143                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
7144                                                                 item_flags, dev,
7145                                                                 error);
7146                         if (ret < 0)
7147                                 return ret;
7148                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
7149                         break;
7150                 case RTE_FLOW_ITEM_TYPE_GENEVE:
7151                         ret = mlx5_flow_validate_item_geneve(items,
7152                                                              item_flags, dev,
7153                                                              error);
7154                         if (ret < 0)
7155                                 return ret;
7156                         geneve_item = items;
7157                         last_item = MLX5_FLOW_LAYER_GENEVE;
7158                         break;
7159                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
7160                         ret = mlx5_flow_validate_item_geneve_opt(items,
7161                                                                  last_item,
7162                                                                  geneve_item,
7163                                                                  dev,
7164                                                                  error);
7165                         if (ret < 0)
7166                                 return ret;
7167                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
7168                         break;
7169                 case RTE_FLOW_ITEM_TYPE_MPLS:
7170                         ret = mlx5_flow_validate_item_mpls(dev, items,
7171                                                            item_flags,
7172                                                            last_item, error);
7173                         if (ret < 0)
7174                                 return ret;
7175                         last_item = MLX5_FLOW_LAYER_MPLS;
7176                         break;
7177
7178                 case RTE_FLOW_ITEM_TYPE_MARK:
7179                         ret = flow_dv_validate_item_mark(dev, items, attr,
7180                                                          error);
7181                         if (ret < 0)
7182                                 return ret;
7183                         last_item = MLX5_FLOW_ITEM_MARK;
7184                         break;
7185                 case RTE_FLOW_ITEM_TYPE_META:
7186                         ret = flow_dv_validate_item_meta(dev, items, attr,
7187                                                          error);
7188                         if (ret < 0)
7189                                 return ret;
7190                         last_item = MLX5_FLOW_ITEM_METADATA;
7191                         break;
7192                 case RTE_FLOW_ITEM_TYPE_ICMP:
7193                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
7194                                                            next_protocol,
7195                                                            error);
7196                         if (ret < 0)
7197                                 return ret;
7198                         last_item = MLX5_FLOW_LAYER_ICMP;
7199                         break;
7200                 case RTE_FLOW_ITEM_TYPE_ICMP6:
7201                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
7202                                                             next_protocol,
7203                                                             error);
7204                         if (ret < 0)
7205                                 return ret;
7206                         item_ipv6_proto = IPPROTO_ICMPV6;
7207                         last_item = MLX5_FLOW_LAYER_ICMP6;
7208                         break;
7209                 case RTE_FLOW_ITEM_TYPE_TAG:
7210                         ret = flow_dv_validate_item_tag(dev, items,
7211                                                         attr, error);
7212                         if (ret < 0)
7213                                 return ret;
7214                         last_item = MLX5_FLOW_ITEM_TAG;
7215                         break;
7216                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
7217                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
7218                         break;
7219                 case RTE_FLOW_ITEM_TYPE_GTP:
7220                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
7221                                                         error);
7222                         if (ret < 0)
7223                                 return ret;
7224                         gtp_item = items;
7225                         last_item = MLX5_FLOW_LAYER_GTP;
7226                         break;
7227                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
7228                         ret = flow_dv_validate_item_gtp_psc(items, last_item,
7229                                                             gtp_item, attr,
7230                                                             error);
7231                         if (ret < 0)
7232                                 return ret;
7233                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
7234                         break;
7235                 case RTE_FLOW_ITEM_TYPE_ECPRI:
7236                         /* Capacity will be checked in the translate stage. */
7237                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
7238                                                             last_item,
7239                                                             ether_type,
7240                                                             &nic_ecpri_mask,
7241                                                             error);
7242                         if (ret < 0)
7243                                 return ret;
7244                         last_item = MLX5_FLOW_LAYER_ECPRI;
7245                         break;
7246                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
7247                         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY)
7248                                 return rte_flow_error_set
7249                                         (error, ENOTSUP,
7250                                          RTE_FLOW_ERROR_TYPE_ITEM,
7251                                          NULL, "multiple integrity items not supported");
7252                         ret = flow_dv_validate_item_integrity(dev, rule_items,
7253                                                               items, error);
7254                         if (ret < 0)
7255                                 return ret;
7256                         last_item = MLX5_FLOW_ITEM_INTEGRITY;
7257                         break;
7258                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
7259                         ret = flow_dv_validate_item_aso_ct(dev, items,
7260                                                            &item_flags, error);
7261                         if (ret < 0)
7262                                 return ret;
7263                         break;
7264                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
7265                         /* tunnel offload item was processed before
7266                          * list it here as a supported type
7267                          */
7268                         break;
7269                 default:
7270                         return rte_flow_error_set(error, ENOTSUP,
7271                                                   RTE_FLOW_ERROR_TYPE_ITEM,
7272                                                   NULL, "item not supported");
7273                 }
7274                 item_flags |= last_item;
7275         }
7276         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7277                 int type = actions->type;
7278                 bool shared_count = false;
7279
7280                 if (!mlx5_flow_os_action_supported(type))
7281                         return rte_flow_error_set(error, ENOTSUP,
7282                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7283                                                   actions,
7284                                                   "action not supported");
7285                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
7286                         return rte_flow_error_set(error, ENOTSUP,
7287                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7288                                                   actions, "too many actions");
7289                 if (action_flags &
7290                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
7291                         return rte_flow_error_set(error, ENOTSUP,
7292                                 RTE_FLOW_ERROR_TYPE_ACTION,
7293                                 NULL, "meter action with policy "
7294                                 "must be the last action");
7295                 switch (type) {
7296                 case RTE_FLOW_ACTION_TYPE_VOID:
7297                         break;
7298                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7299                         ret = flow_dv_validate_action_port_id(dev,
7300                                                               action_flags,
7301                                                               actions,
7302                                                               attr,
7303                                                               error);
7304                         if (ret)
7305                                 return ret;
7306                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7307                         ++actions_n;
7308                         break;
7309                 case RTE_FLOW_ACTION_TYPE_FLAG:
7310                         ret = flow_dv_validate_action_flag(dev, action_flags,
7311                                                            attr, error);
7312                         if (ret < 0)
7313                                 return ret;
7314                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7315                                 /* Count all modify-header actions as one. */
7316                                 if (!(action_flags &
7317                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7318                                         ++actions_n;
7319                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
7320                                                 MLX5_FLOW_ACTION_MARK_EXT;
7321                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7322                                         modify_after_mirror = 1;
7323
7324                         } else {
7325                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
7326                                 ++actions_n;
7327                         }
7328                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7329                         break;
7330                 case RTE_FLOW_ACTION_TYPE_MARK:
7331                         ret = flow_dv_validate_action_mark(dev, actions,
7332                                                            action_flags,
7333                                                            attr, error);
7334                         if (ret < 0)
7335                                 return ret;
7336                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7337                                 /* Count all modify-header actions as one. */
7338                                 if (!(action_flags &
7339                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7340                                         ++actions_n;
7341                                 action_flags |= MLX5_FLOW_ACTION_MARK |
7342                                                 MLX5_FLOW_ACTION_MARK_EXT;
7343                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7344                                         modify_after_mirror = 1;
7345                         } else {
7346                                 action_flags |= MLX5_FLOW_ACTION_MARK;
7347                                 ++actions_n;
7348                         }
7349                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7350                         break;
7351                 case RTE_FLOW_ACTION_TYPE_SET_META:
7352                         ret = flow_dv_validate_action_set_meta(dev, actions,
7353                                                                action_flags,
7354                                                                attr, error);
7355                         if (ret < 0)
7356                                 return ret;
7357                         /* Count all modify-header actions as one action. */
7358                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7359                                 ++actions_n;
7360                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7361                                 modify_after_mirror = 1;
7362                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7363                         rw_act_num += MLX5_ACT_NUM_SET_META;
7364                         break;
7365                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7366                         ret = flow_dv_validate_action_set_tag(dev, actions,
7367                                                               action_flags,
7368                                                               attr, error);
7369                         if (ret < 0)
7370                                 return ret;
7371                         /* Count all modify-header actions as one action. */
7372                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7373                                 ++actions_n;
7374                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7375                                 modify_after_mirror = 1;
7376                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7377                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7378                         break;
7379                 case RTE_FLOW_ACTION_TYPE_DROP:
7380                         ret = mlx5_flow_validate_action_drop(action_flags,
7381                                                              attr, error);
7382                         if (ret < 0)
7383                                 return ret;
7384                         action_flags |= MLX5_FLOW_ACTION_DROP;
7385                         ++actions_n;
7386                         break;
7387                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7388                         ret = mlx5_flow_validate_action_queue(actions,
7389                                                               action_flags, dev,
7390                                                               attr, error);
7391                         if (ret < 0)
7392                                 return ret;
7393                         queue_index = ((const struct rte_flow_action_queue *)
7394                                                         (actions->conf))->index;
7395                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7396                         ++actions_n;
7397                         break;
7398                 case RTE_FLOW_ACTION_TYPE_RSS:
7399                         rss = actions->conf;
7400                         ret = mlx5_flow_validate_action_rss(actions,
7401                                                             action_flags, dev,
7402                                                             attr, item_flags,
7403                                                             error);
7404                         if (ret < 0)
7405                                 return ret;
7406                         if (rss && sample_rss &&
7407                             (sample_rss->level != rss->level ||
7408                             sample_rss->types != rss->types))
7409                                 return rte_flow_error_set(error, ENOTSUP,
7410                                         RTE_FLOW_ERROR_TYPE_ACTION,
7411                                         NULL,
7412                                         "Can't use the different RSS types "
7413                                         "or level in the same flow");
7414                         if (rss != NULL && rss->queue_num)
7415                                 queue_index = rss->queue[0];
7416                         action_flags |= MLX5_FLOW_ACTION_RSS;
7417                         ++actions_n;
7418                         break;
7419                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
7420                         ret =
7421                         mlx5_flow_validate_action_default_miss(action_flags,
7422                                         attr, error);
7423                         if (ret < 0)
7424                                 return ret;
7425                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
7426                         ++actions_n;
7427                         break;
7428                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
7429                 case RTE_FLOW_ACTION_TYPE_COUNT:
7430                         shared_count = is_shared_action_count(actions);
7431                         ret = flow_dv_validate_action_count(dev, shared_count,
7432                                                             action_flags,
7433                                                             error);
7434                         if (ret < 0)
7435                                 return ret;
7436                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7437                         ++actions_n;
7438                         break;
7439                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7440                         if (flow_dv_validate_action_pop_vlan(dev,
7441                                                              action_flags,
7442                                                              actions,
7443                                                              item_flags, attr,
7444                                                              error))
7445                                 return -rte_errno;
7446                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7447                                 modify_after_mirror = 1;
7448                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7449                         ++actions_n;
7450                         break;
7451                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
7452                         ret = flow_dv_validate_action_push_vlan(dev,
7453                                                                 action_flags,
7454                                                                 vlan_m,
7455                                                                 actions, attr,
7456                                                                 error);
7457                         if (ret < 0)
7458                                 return ret;
7459                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7460                                 modify_after_mirror = 1;
7461                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
7462                         ++actions_n;
7463                         break;
7464                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
7465                         ret = flow_dv_validate_action_set_vlan_pcp
7466                                                 (action_flags, actions, error);
7467                         if (ret < 0)
7468                                 return ret;
7469                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7470                                 modify_after_mirror = 1;
7471                         /* Count PCP with push_vlan command. */
7472                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
7473                         break;
7474                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
7475                         ret = flow_dv_validate_action_set_vlan_vid
7476                                                 (item_flags, action_flags,
7477                                                  actions, error);
7478                         if (ret < 0)
7479                                 return ret;
7480                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7481                                 modify_after_mirror = 1;
7482                         /* Count VID with push_vlan command. */
7483                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
7484                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
7485                         break;
7486                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
7487                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
7488                         ret = flow_dv_validate_action_l2_encap(dev,
7489                                                                action_flags,
7490                                                                actions, attr,
7491                                                                error);
7492                         if (ret < 0)
7493                                 return ret;
7494                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
7495                         ++actions_n;
7496                         break;
7497                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
7498                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
7499                         ret = flow_dv_validate_action_decap(dev, action_flags,
7500                                                             actions, item_flags,
7501                                                             attr, error);
7502                         if (ret < 0)
7503                                 return ret;
7504                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7505                                 modify_after_mirror = 1;
7506                         action_flags |= MLX5_FLOW_ACTION_DECAP;
7507                         ++actions_n;
7508                         break;
7509                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
7510                         ret = flow_dv_validate_action_raw_encap_decap
7511                                 (dev, NULL, actions->conf, attr, &action_flags,
7512                                  &actions_n, actions, item_flags, error);
7513                         if (ret < 0)
7514                                 return ret;
7515                         break;
7516                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
7517                         decap = actions->conf;
7518                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
7519                                 ;
7520                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
7521                                 encap = NULL;
7522                                 actions--;
7523                         } else {
7524                                 encap = actions->conf;
7525                         }
7526                         ret = flow_dv_validate_action_raw_encap_decap
7527                                            (dev,
7528                                             decap ? decap : &empty_decap, encap,
7529                                             attr, &action_flags, &actions_n,
7530                                             actions, item_flags, error);
7531                         if (ret < 0)
7532                                 return ret;
7533                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7534                             (action_flags & MLX5_FLOW_ACTION_DECAP))
7535                                 modify_after_mirror = 1;
7536                         break;
7537                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
7538                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
7539                         ret = flow_dv_validate_action_modify_mac(action_flags,
7540                                                                  actions,
7541                                                                  item_flags,
7542                                                                  error);
7543                         if (ret < 0)
7544                                 return ret;
7545                         /* Count all modify-header actions as one action. */
7546                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7547                                 ++actions_n;
7548                         action_flags |= actions->type ==
7549                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
7550                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
7551                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
7552                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7553                                 modify_after_mirror = 1;
7554                         /*
7555                          * Even if the source and destination MAC addresses have
7556                          * overlap in the header with 4B alignment, the convert
7557                          * function will handle them separately and 4 SW actions
7558                          * will be created. And 2 actions will be added each
7559                          * time no matter how many bytes of address will be set.
7560                          */
7561                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
7562                         break;
7563                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
7564                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
7565                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
7566                                                                   actions,
7567                                                                   item_flags,
7568                                                                   error);
7569                         if (ret < 0)
7570                                 return ret;
7571                         /* Count all modify-header actions as one action. */
7572                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7573                                 ++actions_n;
7574                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7575                                 modify_after_mirror = 1;
7576                         action_flags |= actions->type ==
7577                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
7578                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
7579                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
7580                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
7581                         break;
7582                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
7583                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
7584                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
7585                                                                   actions,
7586                                                                   item_flags,
7587                                                                   error);
7588                         if (ret < 0)
7589                                 return ret;
7590                         if (item_ipv6_proto == IPPROTO_ICMPV6)
7591                                 return rte_flow_error_set(error, ENOTSUP,
7592                                         RTE_FLOW_ERROR_TYPE_ACTION,
7593                                         actions,
7594                                         "Can't change header "
7595                                         "with ICMPv6 proto");
7596                         /* Count all modify-header actions as one action. */
7597                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7598                                 ++actions_n;
7599                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7600                                 modify_after_mirror = 1;
7601                         action_flags |= actions->type ==
7602                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
7603                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
7604                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
7605                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
7606                         break;
7607                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
7608                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
7609                         ret = flow_dv_validate_action_modify_tp(action_flags,
7610                                                                 actions,
7611                                                                 item_flags,
7612                                                                 error);
7613                         if (ret < 0)
7614                                 return ret;
7615                         /* Count all modify-header actions as one action. */
7616                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7617                                 ++actions_n;
7618                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7619                                 modify_after_mirror = 1;
7620                         action_flags |= actions->type ==
7621                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
7622                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
7623                                                 MLX5_FLOW_ACTION_SET_TP_DST;
7624                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
7625                         break;
7626                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
7627                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
7628                         ret = flow_dv_validate_action_modify_ttl(action_flags,
7629                                                                  actions,
7630                                                                  item_flags,
7631                                                                  error);
7632                         if (ret < 0)
7633                                 return ret;
7634                         /* Count all modify-header actions as one action. */
7635                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7636                                 ++actions_n;
7637                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7638                                 modify_after_mirror = 1;
7639                         action_flags |= actions->type ==
7640                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
7641                                                 MLX5_FLOW_ACTION_SET_TTL :
7642                                                 MLX5_FLOW_ACTION_DEC_TTL;
7643                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
7644                         break;
7645                 case RTE_FLOW_ACTION_TYPE_JUMP:
7646                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
7647                                                            action_flags,
7648                                                            attr, external,
7649                                                            error);
7650                         if (ret)
7651                                 return ret;
7652                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7653                             fdb_mirror_limit)
7654                                 return rte_flow_error_set(error, EINVAL,
7655                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7656                                                   NULL,
7657                                                   "sample and jump action combination is not supported");
7658                         ++actions_n;
7659                         action_flags |= MLX5_FLOW_ACTION_JUMP;
7660                         break;
7661                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
7662                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
7663                         ret = flow_dv_validate_action_modify_tcp_seq
7664                                                                 (action_flags,
7665                                                                  actions,
7666                                                                  item_flags,
7667                                                                  error);
7668                         if (ret < 0)
7669                                 return ret;
7670                         /* Count all modify-header actions as one action. */
7671                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7672                                 ++actions_n;
7673                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7674                                 modify_after_mirror = 1;
7675                         action_flags |= actions->type ==
7676                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
7677                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
7678                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
7679                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
7680                         break;
7681                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
7682                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
7683                         ret = flow_dv_validate_action_modify_tcp_ack
7684                                                                 (action_flags,
7685                                                                  actions,
7686                                                                  item_flags,
7687                                                                  error);
7688                         if (ret < 0)
7689                                 return ret;
7690                         /* Count all modify-header actions as one action. */
7691                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7692                                 ++actions_n;
7693                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7694                                 modify_after_mirror = 1;
7695                         action_flags |= actions->type ==
7696                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
7697                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
7698                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
7699                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
7700                         break;
7701                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7702                         break;
7703                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
7704                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7705                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7706                         break;
7707                 case RTE_FLOW_ACTION_TYPE_METER:
7708                         ret = mlx5_flow_validate_action_meter(dev,
7709                                                               action_flags,
7710                                                               actions, attr,
7711                                                               port_id_item,
7712                                                               &def_policy,
7713                                                               error);
7714                         if (ret < 0)
7715                                 return ret;
7716                         action_flags |= MLX5_FLOW_ACTION_METER;
7717                         if (!def_policy)
7718                                 action_flags |=
7719                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
7720                         ++actions_n;
7721                         /* Meter action will add one more TAG action. */
7722                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7723                         break;
7724                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
7725                         if (!attr->transfer && !attr->group)
7726                                 return rte_flow_error_set(error, ENOTSUP,
7727                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7728                                                                            NULL,
7729                           "Shared ASO age action is not supported for group 0");
7730                         if (action_flags & MLX5_FLOW_ACTION_AGE)
7731                                 return rte_flow_error_set
7732                                                   (error, EINVAL,
7733                                                    RTE_FLOW_ERROR_TYPE_ACTION,
7734                                                    NULL,
7735                                                    "duplicate age actions set");
7736                         action_flags |= MLX5_FLOW_ACTION_AGE;
7737                         ++actions_n;
7738                         break;
7739                 case RTE_FLOW_ACTION_TYPE_AGE:
7740                         ret = flow_dv_validate_action_age(action_flags,
7741                                                           actions, dev,
7742                                                           error);
7743                         if (ret < 0)
7744                                 return ret;
7745                         /*
7746                          * Validate the regular AGE action (using counter)
7747                          * mutual exclusion with share counter actions.
7748                          */
7749                         if (!priv->sh->flow_hit_aso_en) {
7750                                 if (shared_count)
7751                                         return rte_flow_error_set
7752                                                 (error, EINVAL,
7753                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7754                                                 NULL,
7755                                                 "old age and shared count combination is not supported");
7756                                 if (sample_count)
7757                                         return rte_flow_error_set
7758                                                 (error, EINVAL,
7759                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7760                                                 NULL,
7761                                                 "old age action and count must be in the same sub flow");
7762                         }
7763                         action_flags |= MLX5_FLOW_ACTION_AGE;
7764                         ++actions_n;
7765                         break;
7766                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
7767                         ret = flow_dv_validate_action_modify_ipv4_dscp
7768                                                          (action_flags,
7769                                                           actions,
7770                                                           item_flags,
7771                                                           error);
7772                         if (ret < 0)
7773                                 return ret;
7774                         /* Count all modify-header actions as one action. */
7775                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7776                                 ++actions_n;
7777                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7778                                 modify_after_mirror = 1;
7779                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
7780                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7781                         break;
7782                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
7783                         ret = flow_dv_validate_action_modify_ipv6_dscp
7784                                                                 (action_flags,
7785                                                                  actions,
7786                                                                  item_flags,
7787                                                                  error);
7788                         if (ret < 0)
7789                                 return ret;
7790                         /* Count all modify-header actions as one action. */
7791                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7792                                 ++actions_n;
7793                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7794                                 modify_after_mirror = 1;
7795                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
7796                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7797                         break;
7798                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
7799                         ret = flow_dv_validate_action_sample(&action_flags,
7800                                                              actions, dev,
7801                                                              attr, item_flags,
7802                                                              rss, &sample_rss,
7803                                                              &sample_count,
7804                                                              &fdb_mirror_limit,
7805                                                              error);
7806                         if (ret < 0)
7807                                 return ret;
7808                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
7809                         ++actions_n;
7810                         break;
7811                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
7812                         ret = flow_dv_validate_action_modify_field(dev,
7813                                                                    action_flags,
7814                                                                    actions,
7815                                                                    attr,
7816                                                                    error);
7817                         if (ret < 0)
7818                                 return ret;
7819                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7820                                 modify_after_mirror = 1;
7821                         /* Count all modify-header actions as one action. */
7822                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7823                                 ++actions_n;
7824                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
7825                         rw_act_num += ret;
7826                         break;
7827                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
7828                         ret = flow_dv_validate_action_aso_ct(dev, action_flags,
7829                                                              item_flags, attr,
7830                                                              error);
7831                         if (ret < 0)
7832                                 return ret;
7833                         action_flags |= MLX5_FLOW_ACTION_CT;
7834                         break;
7835                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
7836                         /* tunnel offload action was processed before
7837                          * list it here as a supported type
7838                          */
7839                         break;
7840                 default:
7841                         return rte_flow_error_set(error, ENOTSUP,
7842                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7843                                                   actions,
7844                                                   "action not supported");
7845                 }
7846         }
7847         /*
7848          * Validate actions in flow rules
7849          * - Explicit decap action is prohibited by the tunnel offload API.
7850          * - Drop action in tunnel steer rule is prohibited by the API.
7851          * - Application cannot use MARK action because it's value can mask
7852          *   tunnel default miss nitification.
7853          * - JUMP in tunnel match rule has no support in current PMD
7854          *   implementation.
7855          * - TAG & META are reserved for future uses.
7856          */
7857         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
7858                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
7859                                             MLX5_FLOW_ACTION_MARK     |
7860                                             MLX5_FLOW_ACTION_SET_TAG  |
7861                                             MLX5_FLOW_ACTION_SET_META |
7862                                             MLX5_FLOW_ACTION_DROP;
7863
7864                 if (action_flags & bad_actions_mask)
7865                         return rte_flow_error_set
7866                                         (error, EINVAL,
7867                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7868                                         "Invalid RTE action in tunnel "
7869                                         "set decap rule");
7870                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
7871                         return rte_flow_error_set
7872                                         (error, EINVAL,
7873                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7874                                         "tunnel set decap rule must terminate "
7875                                         "with JUMP");
7876                 if (!attr->ingress)
7877                         return rte_flow_error_set
7878                                         (error, EINVAL,
7879                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7880                                         "tunnel flows for ingress traffic only");
7881         }
7882         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
7883                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
7884                                             MLX5_FLOW_ACTION_MARK    |
7885                                             MLX5_FLOW_ACTION_SET_TAG |
7886                                             MLX5_FLOW_ACTION_SET_META;
7887
7888                 if (action_flags & bad_actions_mask)
7889                         return rte_flow_error_set
7890                                         (error, EINVAL,
7891                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7892                                         "Invalid RTE action in tunnel "
7893                                         "set match rule");
7894         }
7895         /*
7896          * Validate the drop action mutual exclusion with other actions.
7897          * Drop action is mutually-exclusive with any other action, except for
7898          * Count action.
7899          * Drop action compatibility with tunnel offload was already validated.
7900          */
7901         if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH |
7902                             MLX5_FLOW_ACTION_TUNNEL_MATCH));
7903         else if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
7904             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
7905                 return rte_flow_error_set(error, EINVAL,
7906                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7907                                           "Drop action is mutually-exclusive "
7908                                           "with any other action, except for "
7909                                           "Count action");
7910         /* Eswitch has few restrictions on using items and actions */
7911         if (attr->transfer) {
7912                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7913                     action_flags & MLX5_FLOW_ACTION_FLAG)
7914                         return rte_flow_error_set(error, ENOTSUP,
7915                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7916                                                   NULL,
7917                                                   "unsupported action FLAG");
7918                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7919                     action_flags & MLX5_FLOW_ACTION_MARK)
7920                         return rte_flow_error_set(error, ENOTSUP,
7921                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7922                                                   NULL,
7923                                                   "unsupported action MARK");
7924                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
7925                         return rte_flow_error_set(error, ENOTSUP,
7926                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7927                                                   NULL,
7928                                                   "unsupported action QUEUE");
7929                 if (action_flags & MLX5_FLOW_ACTION_RSS)
7930                         return rte_flow_error_set(error, ENOTSUP,
7931                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7932                                                   NULL,
7933                                                   "unsupported action RSS");
7934                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
7935                         return rte_flow_error_set(error, EINVAL,
7936                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7937                                                   actions,
7938                                                   "no fate action is found");
7939         } else {
7940                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
7941                         return rte_flow_error_set(error, EINVAL,
7942                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7943                                                   actions,
7944                                                   "no fate action is found");
7945         }
7946         /*
7947          * Continue validation for Xcap and VLAN actions.
7948          * If hairpin is working in explicit TX rule mode, there is no actions
7949          * splitting and the validation of hairpin ingress flow should be the
7950          * same as other standard flows.
7951          */
7952         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
7953                              MLX5_FLOW_VLAN_ACTIONS)) &&
7954             (queue_index == 0xFFFF ||
7955              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN ||
7956              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
7957              conf->tx_explicit != 0))) {
7958                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
7959                     MLX5_FLOW_XCAP_ACTIONS)
7960                         return rte_flow_error_set(error, ENOTSUP,
7961                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7962                                                   NULL, "encap and decap "
7963                                                   "combination aren't supported");
7964                 if (!attr->transfer && attr->ingress) {
7965                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7966                                 return rte_flow_error_set
7967                                                 (error, ENOTSUP,
7968                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7969                                                  NULL, "encap is not supported"
7970                                                  " for ingress traffic");
7971                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7972                                 return rte_flow_error_set
7973                                                 (error, ENOTSUP,
7974                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7975                                                  NULL, "push VLAN action not "
7976                                                  "supported for ingress");
7977                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
7978                                         MLX5_FLOW_VLAN_ACTIONS)
7979                                 return rte_flow_error_set
7980                                                 (error, ENOTSUP,
7981                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7982                                                  NULL, "no support for "
7983                                                  "multiple VLAN actions");
7984                 }
7985         }
7986         if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) {
7987                 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS &
7988                         ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) &&
7989                         attr->ingress)
7990                         return rte_flow_error_set
7991                                 (error, ENOTSUP,
7992                                 RTE_FLOW_ERROR_TYPE_ACTION,
7993                                 NULL, "fate action not supported for "
7994                                 "meter with policy");
7995                 if (attr->egress) {
7996                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
7997                                 return rte_flow_error_set
7998                                         (error, ENOTSUP,
7999                                         RTE_FLOW_ERROR_TYPE_ACTION,
8000                                         NULL, "modify header action in egress "
8001                                         "cannot be done before meter action");
8002                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
8003                                 return rte_flow_error_set
8004                                         (error, ENOTSUP,
8005                                         RTE_FLOW_ERROR_TYPE_ACTION,
8006                                         NULL, "encap action in egress "
8007                                         "cannot be done before meter action");
8008                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8009                                 return rte_flow_error_set
8010                                         (error, ENOTSUP,
8011                                         RTE_FLOW_ERROR_TYPE_ACTION,
8012                                         NULL, "push vlan action in egress "
8013                                         "cannot be done before meter action");
8014                 }
8015         }
8016         /*
8017          * Hairpin flow will add one more TAG action in TX implicit mode.
8018          * In TX explicit mode, there will be no hairpin flow ID.
8019          */
8020         if (hairpin > 0)
8021                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
8022         /* extra metadata enabled: one more TAG action will be add. */
8023         if (dev_conf->dv_flow_en &&
8024             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
8025             mlx5_flow_ext_mreg_supported(dev))
8026                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
8027         if (rw_act_num >
8028                         flow_dv_modify_hdr_action_max(dev, is_root)) {
8029                 return rte_flow_error_set(error, ENOTSUP,
8030                                           RTE_FLOW_ERROR_TYPE_ACTION,
8031                                           NULL, "too many header modify"
8032                                           " actions to support");
8033         }
8034         /* Eswitch egress mirror and modify flow has limitation on CX5 */
8035         if (fdb_mirror_limit && modify_after_mirror)
8036                 return rte_flow_error_set(error, EINVAL,
8037                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
8038                                 "sample before modify action is not supported");
8039         return 0;
8040 }
8041
8042 /**
8043  * Internal preparation function. Allocates the DV flow size,
8044  * this size is constant.
8045  *
8046  * @param[in] dev
8047  *   Pointer to the rte_eth_dev structure.
8048  * @param[in] attr
8049  *   Pointer to the flow attributes.
8050  * @param[in] items
8051  *   Pointer to the list of items.
8052  * @param[in] actions
8053  *   Pointer to the list of actions.
8054  * @param[out] error
8055  *   Pointer to the error structure.
8056  *
8057  * @return
8058  *   Pointer to mlx5_flow object on success,
8059  *   otherwise NULL and rte_errno is set.
8060  */
8061 static struct mlx5_flow *
8062 flow_dv_prepare(struct rte_eth_dev *dev,
8063                 const struct rte_flow_attr *attr __rte_unused,
8064                 const struct rte_flow_item items[] __rte_unused,
8065                 const struct rte_flow_action actions[] __rte_unused,
8066                 struct rte_flow_error *error)
8067 {
8068         uint32_t handle_idx = 0;
8069         struct mlx5_flow *dev_flow;
8070         struct mlx5_flow_handle *dev_handle;
8071         struct mlx5_priv *priv = dev->data->dev_private;
8072         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
8073
8074         MLX5_ASSERT(wks);
8075         wks->skip_matcher_reg = 0;
8076         wks->policy = NULL;
8077         wks->final_policy = NULL;
8078         /* In case of corrupting the memory. */
8079         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
8080                 rte_flow_error_set(error, ENOSPC,
8081                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8082                                    "not free temporary device flow");
8083                 return NULL;
8084         }
8085         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8086                                    &handle_idx);
8087         if (!dev_handle) {
8088                 rte_flow_error_set(error, ENOMEM,
8089                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8090                                    "not enough memory to create flow handle");
8091                 return NULL;
8092         }
8093         MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows));
8094         dev_flow = &wks->flows[wks->flow_idx++];
8095         memset(dev_flow, 0, sizeof(*dev_flow));
8096         dev_flow->handle = dev_handle;
8097         dev_flow->handle_idx = handle_idx;
8098         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
8099         dev_flow->ingress = attr->ingress;
8100         dev_flow->dv.transfer = attr->transfer;
8101         return dev_flow;
8102 }
8103
8104 #ifdef RTE_LIBRTE_MLX5_DEBUG
8105 /**
8106  * Sanity check for match mask and value. Similar to check_valid_spec() in
8107  * kernel driver. If unmasked bit is present in value, it returns failure.
8108  *
8109  * @param match_mask
8110  *   pointer to match mask buffer.
8111  * @param match_value
8112  *   pointer to match value buffer.
8113  *
8114  * @return
8115  *   0 if valid, -EINVAL otherwise.
8116  */
8117 static int
8118 flow_dv_check_valid_spec(void *match_mask, void *match_value)
8119 {
8120         uint8_t *m = match_mask;
8121         uint8_t *v = match_value;
8122         unsigned int i;
8123
8124         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
8125                 if (v[i] & ~m[i]) {
8126                         DRV_LOG(ERR,
8127                                 "match_value differs from match_criteria"
8128                                 " %p[%u] != %p[%u]",
8129                                 match_value, i, match_mask, i);
8130                         return -EINVAL;
8131                 }
8132         }
8133         return 0;
8134 }
8135 #endif
8136
8137 /**
8138  * Add match of ip_version.
8139  *
8140  * @param[in] group
8141  *   Flow group.
8142  * @param[in] headers_v
8143  *   Values header pointer.
8144  * @param[in] headers_m
8145  *   Masks header pointer.
8146  * @param[in] ip_version
8147  *   The IP version to set.
8148  */
8149 static inline void
8150 flow_dv_set_match_ip_version(uint32_t group,
8151                              void *headers_v,
8152                              void *headers_m,
8153                              uint8_t ip_version)
8154 {
8155         if (group == 0)
8156                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
8157         else
8158                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
8159                          ip_version);
8160         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
8161         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
8162         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
8163 }
8164
8165 /**
8166  * Add Ethernet item to matcher and to the value.
8167  *
8168  * @param[in, out] matcher
8169  *   Flow matcher.
8170  * @param[in, out] key
8171  *   Flow matcher value.
8172  * @param[in] item
8173  *   Flow pattern to translate.
8174  * @param[in] inner
8175  *   Item is inner pattern.
8176  */
8177 static void
8178 flow_dv_translate_item_eth(void *matcher, void *key,
8179                            const struct rte_flow_item *item, int inner,
8180                            uint32_t group)
8181 {
8182         const struct rte_flow_item_eth *eth_m = item->mask;
8183         const struct rte_flow_item_eth *eth_v = item->spec;
8184         const struct rte_flow_item_eth nic_mask = {
8185                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8186                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8187                 .type = RTE_BE16(0xffff),
8188                 .has_vlan = 0,
8189         };
8190         void *hdrs_m;
8191         void *hdrs_v;
8192         char *l24_v;
8193         unsigned int i;
8194
8195         if (!eth_v)
8196                 return;
8197         if (!eth_m)
8198                 eth_m = &nic_mask;
8199         if (inner) {
8200                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8201                                          inner_headers);
8202                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8203         } else {
8204                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8205                                          outer_headers);
8206                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8207         }
8208         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
8209                &eth_m->dst, sizeof(eth_m->dst));
8210         /* The value must be in the range of the mask. */
8211         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
8212         for (i = 0; i < sizeof(eth_m->dst); ++i)
8213                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
8214         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
8215                &eth_m->src, sizeof(eth_m->src));
8216         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
8217         /* The value must be in the range of the mask. */
8218         for (i = 0; i < sizeof(eth_m->dst); ++i)
8219                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
8220         /*
8221          * HW supports match on one Ethertype, the Ethertype following the last
8222          * VLAN tag of the packet (see PRM).
8223          * Set match on ethertype only if ETH header is not followed by VLAN.
8224          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8225          * ethertype, and use ip_version field instead.
8226          * eCPRI over Ether layer will use type value 0xAEFE.
8227          */
8228         if (eth_m->type == 0xFFFF) {
8229                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
8230                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8231                 switch (eth_v->type) {
8232                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8233                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8234                         return;
8235                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
8236                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8237                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8238                         return;
8239                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8240                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8241                         return;
8242                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8243                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8244                         return;
8245                 default:
8246                         break;
8247                 }
8248         }
8249         if (eth_m->has_vlan) {
8250                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8251                 if (eth_v->has_vlan) {
8252                         /*
8253                          * Here, when also has_more_vlan field in VLAN item is
8254                          * not set, only single-tagged packets will be matched.
8255                          */
8256                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8257                         return;
8258                 }
8259         }
8260         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8261                  rte_be_to_cpu_16(eth_m->type));
8262         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
8263         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
8264 }
8265
8266 /**
8267  * Add VLAN item to matcher and to the value.
8268  *
8269  * @param[in, out] dev_flow
8270  *   Flow descriptor.
8271  * @param[in, out] matcher
8272  *   Flow matcher.
8273  * @param[in, out] key
8274  *   Flow matcher value.
8275  * @param[in] item
8276  *   Flow pattern to translate.
8277  * @param[in] inner
8278  *   Item is inner pattern.
8279  */
8280 static void
8281 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
8282                             void *matcher, void *key,
8283                             const struct rte_flow_item *item,
8284                             int inner, uint32_t group)
8285 {
8286         const struct rte_flow_item_vlan *vlan_m = item->mask;
8287         const struct rte_flow_item_vlan *vlan_v = item->spec;
8288         void *hdrs_m;
8289         void *hdrs_v;
8290         uint16_t tci_m;
8291         uint16_t tci_v;
8292
8293         if (inner) {
8294                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8295                                          inner_headers);
8296                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8297         } else {
8298                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8299                                          outer_headers);
8300                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8301                 /*
8302                  * This is workaround, masks are not supported,
8303                  * and pre-validated.
8304                  */
8305                 if (vlan_v)
8306                         dev_flow->handle->vf_vlan.tag =
8307                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
8308         }
8309         /*
8310          * When VLAN item exists in flow, mark packet as tagged,
8311          * even if TCI is not specified.
8312          */
8313         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
8314                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8315                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8316         }
8317         if (!vlan_v)
8318                 return;
8319         if (!vlan_m)
8320                 vlan_m = &rte_flow_item_vlan_mask;
8321         tci_m = rte_be_to_cpu_16(vlan_m->tci);
8322         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
8323         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
8324         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
8325         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
8326         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
8327         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
8328         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
8329         /*
8330          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8331          * ethertype, and use ip_version field instead.
8332          */
8333         if (vlan_m->inner_type == 0xFFFF) {
8334                 switch (vlan_v->inner_type) {
8335                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8336                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8337                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8338                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8339                         return;
8340                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8341                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8342                         return;
8343                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8344                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8345                         return;
8346                 default:
8347                         break;
8348                 }
8349         }
8350         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
8351                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8352                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8353                 /* Only one vlan_tag bit can be set. */
8354                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8355                 return;
8356         }
8357         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8358                  rte_be_to_cpu_16(vlan_m->inner_type));
8359         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
8360                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
8361 }
8362
8363 /**
8364  * Add IPV4 item to matcher and to the value.
8365  *
8366  * @param[in, out] matcher
8367  *   Flow matcher.
8368  * @param[in, out] key
8369  *   Flow matcher value.
8370  * @param[in] item
8371  *   Flow pattern to translate.
8372  * @param[in] inner
8373  *   Item is inner pattern.
8374  * @param[in] group
8375  *   The group to insert the rule.
8376  */
8377 static void
8378 flow_dv_translate_item_ipv4(void *matcher, void *key,
8379                             const struct rte_flow_item *item,
8380                             int inner, uint32_t group)
8381 {
8382         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
8383         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
8384         const struct rte_flow_item_ipv4 nic_mask = {
8385                 .hdr = {
8386                         .src_addr = RTE_BE32(0xffffffff),
8387                         .dst_addr = RTE_BE32(0xffffffff),
8388                         .type_of_service = 0xff,
8389                         .next_proto_id = 0xff,
8390                         .time_to_live = 0xff,
8391                 },
8392         };
8393         void *headers_m;
8394         void *headers_v;
8395         char *l24_m;
8396         char *l24_v;
8397         uint8_t tos, ihl_m, ihl_v;
8398
8399         if (inner) {
8400                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8401                                          inner_headers);
8402                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8403         } else {
8404                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8405                                          outer_headers);
8406                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8407         }
8408         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
8409         if (!ipv4_v)
8410                 return;
8411         if (!ipv4_m)
8412                 ipv4_m = &nic_mask;
8413         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8414                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8415         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8416                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8417         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
8418         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
8419         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8420                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8421         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8422                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8423         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
8424         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
8425         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
8426         ihl_m = ipv4_m->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8427         ihl_v = ipv4_v->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8428         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_ihl, ihl_m);
8429         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_ihl, ihl_m & ihl_v);
8430         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
8431                  ipv4_m->hdr.type_of_service);
8432         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
8433         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
8434                  ipv4_m->hdr.type_of_service >> 2);
8435         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
8436         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8437                  ipv4_m->hdr.next_proto_id);
8438         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8439                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
8440         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8441                  ipv4_m->hdr.time_to_live);
8442         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8443                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
8444         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8445                  !!(ipv4_m->hdr.fragment_offset));
8446         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8447                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
8448 }
8449
8450 /**
8451  * Add IPV6 item to matcher and to the value.
8452  *
8453  * @param[in, out] matcher
8454  *   Flow matcher.
8455  * @param[in, out] key
8456  *   Flow matcher value.
8457  * @param[in] item
8458  *   Flow pattern to translate.
8459  * @param[in] inner
8460  *   Item is inner pattern.
8461  * @param[in] group
8462  *   The group to insert the rule.
8463  */
8464 static void
8465 flow_dv_translate_item_ipv6(void *matcher, void *key,
8466                             const struct rte_flow_item *item,
8467                             int inner, uint32_t group)
8468 {
8469         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
8470         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
8471         const struct rte_flow_item_ipv6 nic_mask = {
8472                 .hdr = {
8473                         .src_addr =
8474                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8475                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8476                         .dst_addr =
8477                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8478                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8479                         .vtc_flow = RTE_BE32(0xffffffff),
8480                         .proto = 0xff,
8481                         .hop_limits = 0xff,
8482                 },
8483         };
8484         void *headers_m;
8485         void *headers_v;
8486         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8487         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8488         char *l24_m;
8489         char *l24_v;
8490         uint32_t vtc_m;
8491         uint32_t vtc_v;
8492         int i;
8493         int size;
8494
8495         if (inner) {
8496                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8497                                          inner_headers);
8498                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8499         } else {
8500                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8501                                          outer_headers);
8502                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8503         }
8504         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
8505         if (!ipv6_v)
8506                 return;
8507         if (!ipv6_m)
8508                 ipv6_m = &nic_mask;
8509         size = sizeof(ipv6_m->hdr.dst_addr);
8510         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8511                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8512         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8513                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8514         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
8515         for (i = 0; i < size; ++i)
8516                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
8517         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8518                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8519         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8520                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8521         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
8522         for (i = 0; i < size; ++i)
8523                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
8524         /* TOS. */
8525         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
8526         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
8527         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
8528         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
8529         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
8530         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
8531         /* Label. */
8532         if (inner) {
8533                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
8534                          vtc_m);
8535                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
8536                          vtc_v);
8537         } else {
8538                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
8539                          vtc_m);
8540                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
8541                          vtc_v);
8542         }
8543         /* Protocol. */
8544         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8545                  ipv6_m->hdr.proto);
8546         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8547                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
8548         /* Hop limit. */
8549         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8550                  ipv6_m->hdr.hop_limits);
8551         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8552                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
8553         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8554                  !!(ipv6_m->has_frag_ext));
8555         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8556                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
8557 }
8558
8559 /**
8560  * Add IPV6 fragment extension item to matcher and to the value.
8561  *
8562  * @param[in, out] matcher
8563  *   Flow matcher.
8564  * @param[in, out] key
8565  *   Flow matcher value.
8566  * @param[in] item
8567  *   Flow pattern to translate.
8568  * @param[in] inner
8569  *   Item is inner pattern.
8570  */
8571 static void
8572 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
8573                                      const struct rte_flow_item *item,
8574                                      int inner)
8575 {
8576         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
8577         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
8578         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
8579                 .hdr = {
8580                         .next_header = 0xff,
8581                         .frag_data = RTE_BE16(0xffff),
8582                 },
8583         };
8584         void *headers_m;
8585         void *headers_v;
8586
8587         if (inner) {
8588                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8589                                          inner_headers);
8590                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8591         } else {
8592                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8593                                          outer_headers);
8594                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8595         }
8596         /* IPv6 fragment extension item exists, so packet is IP fragment. */
8597         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
8598         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
8599         if (!ipv6_frag_ext_v)
8600                 return;
8601         if (!ipv6_frag_ext_m)
8602                 ipv6_frag_ext_m = &nic_mask;
8603         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8604                  ipv6_frag_ext_m->hdr.next_header);
8605         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8606                  ipv6_frag_ext_v->hdr.next_header &
8607                  ipv6_frag_ext_m->hdr.next_header);
8608 }
8609
8610 /**
8611  * Add TCP item to matcher and to the value.
8612  *
8613  * @param[in, out] matcher
8614  *   Flow matcher.
8615  * @param[in, out] key
8616  *   Flow matcher value.
8617  * @param[in] item
8618  *   Flow pattern to translate.
8619  * @param[in] inner
8620  *   Item is inner pattern.
8621  */
8622 static void
8623 flow_dv_translate_item_tcp(void *matcher, void *key,
8624                            const struct rte_flow_item *item,
8625                            int inner)
8626 {
8627         const struct rte_flow_item_tcp *tcp_m = item->mask;
8628         const struct rte_flow_item_tcp *tcp_v = item->spec;
8629         void *headers_m;
8630         void *headers_v;
8631
8632         if (inner) {
8633                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8634                                          inner_headers);
8635                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8636         } else {
8637                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8638                                          outer_headers);
8639                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8640         }
8641         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8642         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
8643         if (!tcp_v)
8644                 return;
8645         if (!tcp_m)
8646                 tcp_m = &rte_flow_item_tcp_mask;
8647         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
8648                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
8649         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
8650                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
8651         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
8652                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
8653         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
8654                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
8655         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
8656                  tcp_m->hdr.tcp_flags);
8657         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
8658                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
8659 }
8660
8661 /**
8662  * Add UDP item to matcher and to the value.
8663  *
8664  * @param[in, out] matcher
8665  *   Flow matcher.
8666  * @param[in, out] key
8667  *   Flow matcher value.
8668  * @param[in] item
8669  *   Flow pattern to translate.
8670  * @param[in] inner
8671  *   Item is inner pattern.
8672  */
8673 static void
8674 flow_dv_translate_item_udp(void *matcher, void *key,
8675                            const struct rte_flow_item *item,
8676                            int inner)
8677 {
8678         const struct rte_flow_item_udp *udp_m = item->mask;
8679         const struct rte_flow_item_udp *udp_v = item->spec;
8680         void *headers_m;
8681         void *headers_v;
8682
8683         if (inner) {
8684                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8685                                          inner_headers);
8686                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8687         } else {
8688                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8689                                          outer_headers);
8690                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8691         }
8692         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8693         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
8694         if (!udp_v)
8695                 return;
8696         if (!udp_m)
8697                 udp_m = &rte_flow_item_udp_mask;
8698         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
8699                  rte_be_to_cpu_16(udp_m->hdr.src_port));
8700         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
8701                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
8702         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
8703                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
8704         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
8705                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
8706 }
8707
8708 /**
8709  * Add GRE optional Key item to matcher and to the value.
8710  *
8711  * @param[in, out] matcher
8712  *   Flow matcher.
8713  * @param[in, out] key
8714  *   Flow matcher value.
8715  * @param[in] item
8716  *   Flow pattern to translate.
8717  * @param[in] inner
8718  *   Item is inner pattern.
8719  */
8720 static void
8721 flow_dv_translate_item_gre_key(void *matcher, void *key,
8722                                    const struct rte_flow_item *item)
8723 {
8724         const rte_be32_t *key_m = item->mask;
8725         const rte_be32_t *key_v = item->spec;
8726         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8727         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8728         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
8729
8730         /* GRE K bit must be on and should already be validated */
8731         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
8732         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
8733         if (!key_v)
8734                 return;
8735         if (!key_m)
8736                 key_m = &gre_key_default_mask;
8737         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
8738                  rte_be_to_cpu_32(*key_m) >> 8);
8739         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
8740                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
8741         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
8742                  rte_be_to_cpu_32(*key_m) & 0xFF);
8743         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
8744                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
8745 }
8746
8747 /**
8748  * Add GRE item to matcher and to the value.
8749  *
8750  * @param[in, out] matcher
8751  *   Flow matcher.
8752  * @param[in, out] key
8753  *   Flow matcher value.
8754  * @param[in] item
8755  *   Flow pattern to translate.
8756  * @param[in] inner
8757  *   Item is inner pattern.
8758  */
8759 static void
8760 flow_dv_translate_item_gre(void *matcher, void *key,
8761                            const struct rte_flow_item *item,
8762                            int inner)
8763 {
8764         const struct rte_flow_item_gre *gre_m = item->mask;
8765         const struct rte_flow_item_gre *gre_v = item->spec;
8766         void *headers_m;
8767         void *headers_v;
8768         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8769         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8770         struct {
8771                 union {
8772                         __extension__
8773                         struct {
8774                                 uint16_t version:3;
8775                                 uint16_t rsvd0:9;
8776                                 uint16_t s_present:1;
8777                                 uint16_t k_present:1;
8778                                 uint16_t rsvd_bit1:1;
8779                                 uint16_t c_present:1;
8780                         };
8781                         uint16_t value;
8782                 };
8783         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
8784
8785         if (inner) {
8786                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8787                                          inner_headers);
8788                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8789         } else {
8790                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8791                                          outer_headers);
8792                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8793         }
8794         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8795         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
8796         if (!gre_v)
8797                 return;
8798         if (!gre_m)
8799                 gre_m = &rte_flow_item_gre_mask;
8800         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
8801                  rte_be_to_cpu_16(gre_m->protocol));
8802         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
8803                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
8804         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
8805         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
8806         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
8807                  gre_crks_rsvd0_ver_m.c_present);
8808         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
8809                  gre_crks_rsvd0_ver_v.c_present &
8810                  gre_crks_rsvd0_ver_m.c_present);
8811         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
8812                  gre_crks_rsvd0_ver_m.k_present);
8813         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
8814                  gre_crks_rsvd0_ver_v.k_present &
8815                  gre_crks_rsvd0_ver_m.k_present);
8816         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
8817                  gre_crks_rsvd0_ver_m.s_present);
8818         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
8819                  gre_crks_rsvd0_ver_v.s_present &
8820                  gre_crks_rsvd0_ver_m.s_present);
8821 }
8822
8823 /**
8824  * Add NVGRE item to matcher and to the value.
8825  *
8826  * @param[in, out] matcher
8827  *   Flow matcher.
8828  * @param[in, out] key
8829  *   Flow matcher value.
8830  * @param[in] item
8831  *   Flow pattern to translate.
8832  * @param[in] inner
8833  *   Item is inner pattern.
8834  */
8835 static void
8836 flow_dv_translate_item_nvgre(void *matcher, void *key,
8837                              const struct rte_flow_item *item,
8838                              int inner)
8839 {
8840         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
8841         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
8842         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8843         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8844         const char *tni_flow_id_m;
8845         const char *tni_flow_id_v;
8846         char *gre_key_m;
8847         char *gre_key_v;
8848         int size;
8849         int i;
8850
8851         /* For NVGRE, GRE header fields must be set with defined values. */
8852         const struct rte_flow_item_gre gre_spec = {
8853                 .c_rsvd0_ver = RTE_BE16(0x2000),
8854                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
8855         };
8856         const struct rte_flow_item_gre gre_mask = {
8857                 .c_rsvd0_ver = RTE_BE16(0xB000),
8858                 .protocol = RTE_BE16(UINT16_MAX),
8859         };
8860         const struct rte_flow_item gre_item = {
8861                 .spec = &gre_spec,
8862                 .mask = &gre_mask,
8863                 .last = NULL,
8864         };
8865         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
8866         if (!nvgre_v)
8867                 return;
8868         if (!nvgre_m)
8869                 nvgre_m = &rte_flow_item_nvgre_mask;
8870         tni_flow_id_m = (const char *)nvgre_m->tni;
8871         tni_flow_id_v = (const char *)nvgre_v->tni;
8872         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
8873         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
8874         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
8875         memcpy(gre_key_m, tni_flow_id_m, size);
8876         for (i = 0; i < size; ++i)
8877                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
8878 }
8879
8880 /**
8881  * Add VXLAN item to matcher and to the value.
8882  *
8883  * @param[in] dev
8884  *   Pointer to the Ethernet device structure.
8885  * @param[in] attr
8886  *   Flow rule attributes.
8887  * @param[in, out] matcher
8888  *   Flow matcher.
8889  * @param[in, out] key
8890  *   Flow matcher value.
8891  * @param[in] item
8892  *   Flow pattern to translate.
8893  * @param[in] inner
8894  *   Item is inner pattern.
8895  */
8896 static void
8897 flow_dv_translate_item_vxlan(struct rte_eth_dev *dev,
8898                              const struct rte_flow_attr *attr,
8899                              void *matcher, void *key,
8900                              const struct rte_flow_item *item,
8901                              int inner)
8902 {
8903         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
8904         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
8905         void *headers_m;
8906         void *headers_v;
8907         void *misc5_m;
8908         void *misc5_v;
8909         uint32_t *tunnel_header_v;
8910         uint32_t *tunnel_header_m;
8911         uint16_t dport;
8912         struct mlx5_priv *priv = dev->data->dev_private;
8913         const struct rte_flow_item_vxlan nic_mask = {
8914                 .vni = "\xff\xff\xff",
8915                 .rsvd1 = 0xff,
8916         };
8917
8918         if (inner) {
8919                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8920                                          inner_headers);
8921                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8922         } else {
8923                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8924                                          outer_headers);
8925                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8926         }
8927         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8928                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8929         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8930                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8931                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8932         }
8933         dport = MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport);
8934         if (!vxlan_v)
8935                 return;
8936         if (!vxlan_m) {
8937                 if ((!attr->group && !priv->sh->tunnel_header_0_1) ||
8938                     (attr->group && !priv->sh->misc5_cap))
8939                         vxlan_m = &rte_flow_item_vxlan_mask;
8940                 else
8941                         vxlan_m = &nic_mask;
8942         }
8943         if ((priv->sh->steering_format_version ==
8944             MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 &&
8945             dport != MLX5_UDP_PORT_VXLAN) ||
8946             (!attr->group && !attr->transfer && !priv->sh->tunnel_header_0_1) ||
8947             ((attr->group || attr->transfer) && !priv->sh->misc5_cap)) {
8948                 void *misc_m;
8949                 void *misc_v;
8950                 char *vni_m;
8951                 char *vni_v;
8952                 int size;
8953                 int i;
8954                 misc_m = MLX5_ADDR_OF(fte_match_param,
8955                                       matcher, misc_parameters);
8956                 misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8957                 size = sizeof(vxlan_m->vni);
8958                 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
8959                 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
8960                 memcpy(vni_m, vxlan_m->vni, size);
8961                 for (i = 0; i < size; ++i)
8962                         vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8963                 return;
8964         }
8965         misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
8966         misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
8967         tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8968                                                    misc5_v,
8969                                                    tunnel_header_1);
8970         tunnel_header_m = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8971                                                    misc5_m,
8972                                                    tunnel_header_1);
8973         *tunnel_header_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) |
8974                            (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 |
8975                            (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16;
8976         if (*tunnel_header_v)
8977                 *tunnel_header_m = vxlan_m->vni[0] |
8978                         vxlan_m->vni[1] << 8 |
8979                         vxlan_m->vni[2] << 16;
8980         else
8981                 *tunnel_header_m = 0x0;
8982         *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24;
8983         if (vxlan_v->rsvd1 & vxlan_m->rsvd1)
8984                 *tunnel_header_m |= vxlan_m->rsvd1 << 24;
8985 }
8986
8987 /**
8988  * Add VXLAN-GPE item to matcher and to the value.
8989  *
8990  * @param[in, out] matcher
8991  *   Flow matcher.
8992  * @param[in, out] key
8993  *   Flow matcher value.
8994  * @param[in] item
8995  *   Flow pattern to translate.
8996  * @param[in] inner
8997  *   Item is inner pattern.
8998  */
8999
9000 static void
9001 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
9002                                  const struct rte_flow_item *item, int inner)
9003 {
9004         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
9005         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
9006         void *headers_m;
9007         void *headers_v;
9008         void *misc_m =
9009                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
9010         void *misc_v =
9011                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9012         char *vni_m;
9013         char *vni_v;
9014         uint16_t dport;
9015         int size;
9016         int i;
9017         uint8_t flags_m = 0xff;
9018         uint8_t flags_v = 0xc;
9019
9020         if (inner) {
9021                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9022                                          inner_headers);
9023                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9024         } else {
9025                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9026                                          outer_headers);
9027                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9028         }
9029         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
9030                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
9031         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9032                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9033                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9034         }
9035         if (!vxlan_v)
9036                 return;
9037         if (!vxlan_m)
9038                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
9039         size = sizeof(vxlan_m->vni);
9040         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
9041         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
9042         memcpy(vni_m, vxlan_m->vni, size);
9043         for (i = 0; i < size; ++i)
9044                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
9045         if (vxlan_m->flags) {
9046                 flags_m = vxlan_m->flags;
9047                 flags_v = vxlan_v->flags;
9048         }
9049         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
9050         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
9051         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
9052                  vxlan_m->protocol);
9053         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
9054                  vxlan_v->protocol);
9055 }
9056
9057 /**
9058  * Add Geneve item to matcher and to the value.
9059  *
9060  * @param[in, out] matcher
9061  *   Flow matcher.
9062  * @param[in, out] key
9063  *   Flow matcher value.
9064  * @param[in] item
9065  *   Flow pattern to translate.
9066  * @param[in] inner
9067  *   Item is inner pattern.
9068  */
9069
9070 static void
9071 flow_dv_translate_item_geneve(void *matcher, void *key,
9072                               const struct rte_flow_item *item, int inner)
9073 {
9074         const struct rte_flow_item_geneve *geneve_m = item->mask;
9075         const struct rte_flow_item_geneve *geneve_v = item->spec;
9076         void *headers_m;
9077         void *headers_v;
9078         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9079         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9080         uint16_t dport;
9081         uint16_t gbhdr_m;
9082         uint16_t gbhdr_v;
9083         char *vni_m;
9084         char *vni_v;
9085         size_t size, i;
9086
9087         if (inner) {
9088                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9089                                          inner_headers);
9090                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9091         } else {
9092                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9093                                          outer_headers);
9094                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9095         }
9096         dport = MLX5_UDP_PORT_GENEVE;
9097         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9098                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9099                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9100         }
9101         if (!geneve_v)
9102                 return;
9103         if (!geneve_m)
9104                 geneve_m = &rte_flow_item_geneve_mask;
9105         size = sizeof(geneve_m->vni);
9106         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
9107         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
9108         memcpy(vni_m, geneve_m->vni, size);
9109         for (i = 0; i < size; ++i)
9110                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
9111         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
9112                  rte_be_to_cpu_16(geneve_m->protocol));
9113         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
9114                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
9115         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
9116         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
9117         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
9118                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9119         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
9120                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9121         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9122                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9123         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9124                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
9125                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9126 }
9127
9128 /**
9129  * Create Geneve TLV option resource.
9130  *
9131  * @param dev[in, out]
9132  *   Pointer to rte_eth_dev structure.
9133  * @param[in, out] tag_be24
9134  *   Tag value in big endian then R-shift 8.
9135  * @parm[in, out] dev_flow
9136  *   Pointer to the dev_flow.
9137  * @param[out] error
9138  *   pointer to error structure.
9139  *
9140  * @return
9141  *   0 on success otherwise -errno and errno is set.
9142  */
9143
9144 int
9145 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
9146                                              const struct rte_flow_item *item,
9147                                              struct rte_flow_error *error)
9148 {
9149         struct mlx5_priv *priv = dev->data->dev_private;
9150         struct mlx5_dev_ctx_shared *sh = priv->sh;
9151         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
9152                         sh->geneve_tlv_option_resource;
9153         struct mlx5_devx_obj *obj;
9154         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9155         int ret = 0;
9156
9157         if (!geneve_opt_v)
9158                 return -1;
9159         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
9160         if (geneve_opt_resource != NULL) {
9161                 if (geneve_opt_resource->option_class ==
9162                         geneve_opt_v->option_class &&
9163                         geneve_opt_resource->option_type ==
9164                         geneve_opt_v->option_type &&
9165                         geneve_opt_resource->length ==
9166                         geneve_opt_v->option_len) {
9167                         /* We already have GENVE TLV option obj allocated. */
9168                         __atomic_fetch_add(&geneve_opt_resource->refcnt, 1,
9169                                            __ATOMIC_RELAXED);
9170                 } else {
9171                         ret = rte_flow_error_set(error, ENOMEM,
9172                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9173                                 "Only one GENEVE TLV option supported");
9174                         goto exit;
9175                 }
9176         } else {
9177                 /* Create a GENEVE TLV object and resource. */
9178                 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->ctx,
9179                                 geneve_opt_v->option_class,
9180                                 geneve_opt_v->option_type,
9181                                 geneve_opt_v->option_len);
9182                 if (!obj) {
9183                         ret = rte_flow_error_set(error, ENODATA,
9184                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9185                                 "Failed to create GENEVE TLV Devx object");
9186                         goto exit;
9187                 }
9188                 sh->geneve_tlv_option_resource =
9189                                 mlx5_malloc(MLX5_MEM_ZERO,
9190                                                 sizeof(*geneve_opt_resource),
9191                                                 0, SOCKET_ID_ANY);
9192                 if (!sh->geneve_tlv_option_resource) {
9193                         claim_zero(mlx5_devx_cmd_destroy(obj));
9194                         ret = rte_flow_error_set(error, ENOMEM,
9195                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9196                                 "GENEVE TLV object memory allocation failed");
9197                         goto exit;
9198                 }
9199                 geneve_opt_resource = sh->geneve_tlv_option_resource;
9200                 geneve_opt_resource->obj = obj;
9201                 geneve_opt_resource->option_class = geneve_opt_v->option_class;
9202                 geneve_opt_resource->option_type = geneve_opt_v->option_type;
9203                 geneve_opt_resource->length = geneve_opt_v->option_len;
9204                 __atomic_store_n(&geneve_opt_resource->refcnt, 1,
9205                                 __ATOMIC_RELAXED);
9206         }
9207 exit:
9208         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
9209         return ret;
9210 }
9211
9212 /**
9213  * Add Geneve TLV option item to matcher.
9214  *
9215  * @param[in, out] dev
9216  *   Pointer to rte_eth_dev structure.
9217  * @param[in, out] matcher
9218  *   Flow matcher.
9219  * @param[in, out] key
9220  *   Flow matcher value.
9221  * @param[in] item
9222  *   Flow pattern to translate.
9223  * @param[out] error
9224  *   Pointer to error structure.
9225  */
9226 static int
9227 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher,
9228                                   void *key, const struct rte_flow_item *item,
9229                                   struct rte_flow_error *error)
9230 {
9231         const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask;
9232         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9233         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9234         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9235         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9236                         misc_parameters_3);
9237         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9238         rte_be32_t opt_data_key = 0, opt_data_mask = 0;
9239         int ret = 0;
9240
9241         if (!geneve_opt_v)
9242                 return -1;
9243         if (!geneve_opt_m)
9244                 geneve_opt_m = &rte_flow_item_geneve_opt_mask;
9245         ret = flow_dev_geneve_tlv_option_resource_register(dev, item,
9246                                                            error);
9247         if (ret) {
9248                 DRV_LOG(ERR, "Failed to create geneve_tlv_obj");
9249                 return ret;
9250         }
9251         /*
9252          * Set the option length in GENEVE header if not requested.
9253          * The GENEVE TLV option length is expressed by the option length field
9254          * in the GENEVE header.
9255          * If the option length was not requested but the GENEVE TLV option item
9256          * is present we set the option length field implicitly.
9257          */
9258         if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) {
9259                 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9260                          MLX5_GENEVE_OPTLEN_MASK);
9261                 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9262                          geneve_opt_v->option_len + 1);
9263         }
9264         MLX5_SET(fte_match_set_misc, misc_m, geneve_tlv_option_0_exist, 1);
9265         MLX5_SET(fte_match_set_misc, misc_v, geneve_tlv_option_0_exist, 1);
9266         /* Set the data. */
9267         if (geneve_opt_v->data) {
9268                 memcpy(&opt_data_key, geneve_opt_v->data,
9269                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9270                                 sizeof(opt_data_key)));
9271                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9272                                 sizeof(opt_data_key));
9273                 memcpy(&opt_data_mask, geneve_opt_m->data,
9274                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9275                                 sizeof(opt_data_mask)));
9276                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9277                                 sizeof(opt_data_mask));
9278                 MLX5_SET(fte_match_set_misc3, misc3_m,
9279                                 geneve_tlv_option_0_data,
9280                                 rte_be_to_cpu_32(opt_data_mask));
9281                 MLX5_SET(fte_match_set_misc3, misc3_v,
9282                                 geneve_tlv_option_0_data,
9283                         rte_be_to_cpu_32(opt_data_key & opt_data_mask));
9284         }
9285         return ret;
9286 }
9287
9288 /**
9289  * Add MPLS item to matcher and to the value.
9290  *
9291  * @param[in, out] matcher
9292  *   Flow matcher.
9293  * @param[in, out] key
9294  *   Flow matcher value.
9295  * @param[in] item
9296  *   Flow pattern to translate.
9297  * @param[in] prev_layer
9298  *   The protocol layer indicated in previous item.
9299  * @param[in] inner
9300  *   Item is inner pattern.
9301  */
9302 static void
9303 flow_dv_translate_item_mpls(void *matcher, void *key,
9304                             const struct rte_flow_item *item,
9305                             uint64_t prev_layer,
9306                             int inner)
9307 {
9308         const uint32_t *in_mpls_m = item->mask;
9309         const uint32_t *in_mpls_v = item->spec;
9310         uint32_t *out_mpls_m = 0;
9311         uint32_t *out_mpls_v = 0;
9312         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9313         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9314         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
9315                                      misc_parameters_2);
9316         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9317         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9318         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9319
9320         switch (prev_layer) {
9321         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9322                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
9323                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9324                          MLX5_UDP_PORT_MPLS);
9325                 break;
9326         case MLX5_FLOW_LAYER_GRE:
9327                 /* Fall-through. */
9328         case MLX5_FLOW_LAYER_GRE_KEY:
9329                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
9330                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
9331                          RTE_ETHER_TYPE_MPLS);
9332                 break;
9333         default:
9334                 break;
9335         }
9336         if (!in_mpls_v)
9337                 return;
9338         if (!in_mpls_m)
9339                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
9340         switch (prev_layer) {
9341         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9342                 out_mpls_m =
9343                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9344                                                  outer_first_mpls_over_udp);
9345                 out_mpls_v =
9346                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9347                                                  outer_first_mpls_over_udp);
9348                 break;
9349         case MLX5_FLOW_LAYER_GRE:
9350                 out_mpls_m =
9351                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9352                                                  outer_first_mpls_over_gre);
9353                 out_mpls_v =
9354                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9355                                                  outer_first_mpls_over_gre);
9356                 break;
9357         default:
9358                 /* Inner MPLS not over GRE is not supported. */
9359                 if (!inner) {
9360                         out_mpls_m =
9361                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9362                                                          misc2_m,
9363                                                          outer_first_mpls);
9364                         out_mpls_v =
9365                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9366                                                          misc2_v,
9367                                                          outer_first_mpls);
9368                 }
9369                 break;
9370         }
9371         if (out_mpls_m && out_mpls_v) {
9372                 *out_mpls_m = *in_mpls_m;
9373                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
9374         }
9375 }
9376
9377 /**
9378  * Add metadata register item to matcher
9379  *
9380  * @param[in, out] matcher
9381  *   Flow matcher.
9382  * @param[in, out] key
9383  *   Flow matcher value.
9384  * @param[in] reg_type
9385  *   Type of device metadata register
9386  * @param[in] value
9387  *   Register value
9388  * @param[in] mask
9389  *   Register mask
9390  */
9391 static void
9392 flow_dv_match_meta_reg(void *matcher, void *key,
9393                        enum modify_reg reg_type,
9394                        uint32_t data, uint32_t mask)
9395 {
9396         void *misc2_m =
9397                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
9398         void *misc2_v =
9399                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9400         uint32_t temp;
9401
9402         data &= mask;
9403         switch (reg_type) {
9404         case REG_A:
9405                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
9406                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
9407                 break;
9408         case REG_B:
9409                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
9410                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
9411                 break;
9412         case REG_C_0:
9413                 /*
9414                  * The metadata register C0 field might be divided into
9415                  * source vport index and META item value, we should set
9416                  * this field according to specified mask, not as whole one.
9417                  */
9418                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
9419                 temp |= mask;
9420                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
9421                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
9422                 temp &= ~mask;
9423                 temp |= data;
9424                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
9425                 break;
9426         case REG_C_1:
9427                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
9428                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
9429                 break;
9430         case REG_C_2:
9431                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
9432                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
9433                 break;
9434         case REG_C_3:
9435                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
9436                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
9437                 break;
9438         case REG_C_4:
9439                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
9440                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
9441                 break;
9442         case REG_C_5:
9443                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
9444                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
9445                 break;
9446         case REG_C_6:
9447                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
9448                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
9449                 break;
9450         case REG_C_7:
9451                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
9452                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
9453                 break;
9454         default:
9455                 MLX5_ASSERT(false);
9456                 break;
9457         }
9458 }
9459
9460 /**
9461  * Add MARK item to matcher
9462  *
9463  * @param[in] dev
9464  *   The device to configure through.
9465  * @param[in, out] matcher
9466  *   Flow matcher.
9467  * @param[in, out] key
9468  *   Flow matcher value.
9469  * @param[in] item
9470  *   Flow pattern to translate.
9471  */
9472 static void
9473 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
9474                             void *matcher, void *key,
9475                             const struct rte_flow_item *item)
9476 {
9477         struct mlx5_priv *priv = dev->data->dev_private;
9478         const struct rte_flow_item_mark *mark;
9479         uint32_t value;
9480         uint32_t mask;
9481
9482         mark = item->mask ? (const void *)item->mask :
9483                             &rte_flow_item_mark_mask;
9484         mask = mark->id & priv->sh->dv_mark_mask;
9485         mark = (const void *)item->spec;
9486         MLX5_ASSERT(mark);
9487         value = mark->id & priv->sh->dv_mark_mask & mask;
9488         if (mask) {
9489                 enum modify_reg reg;
9490
9491                 /* Get the metadata register index for the mark. */
9492                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
9493                 MLX5_ASSERT(reg > 0);
9494                 if (reg == REG_C_0) {
9495                         struct mlx5_priv *priv = dev->data->dev_private;
9496                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9497                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9498
9499                         mask &= msk_c0;
9500                         mask <<= shl_c0;
9501                         value <<= shl_c0;
9502                 }
9503                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9504         }
9505 }
9506
9507 /**
9508  * Add META item to matcher
9509  *
9510  * @param[in] dev
9511  *   The devich to configure through.
9512  * @param[in, out] matcher
9513  *   Flow matcher.
9514  * @param[in, out] key
9515  *   Flow matcher value.
9516  * @param[in] attr
9517  *   Attributes of flow that includes this item.
9518  * @param[in] item
9519  *   Flow pattern to translate.
9520  */
9521 static void
9522 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
9523                             void *matcher, void *key,
9524                             const struct rte_flow_attr *attr,
9525                             const struct rte_flow_item *item)
9526 {
9527         const struct rte_flow_item_meta *meta_m;
9528         const struct rte_flow_item_meta *meta_v;
9529
9530         meta_m = (const void *)item->mask;
9531         if (!meta_m)
9532                 meta_m = &rte_flow_item_meta_mask;
9533         meta_v = (const void *)item->spec;
9534         if (meta_v) {
9535                 int reg;
9536                 uint32_t value = meta_v->data;
9537                 uint32_t mask = meta_m->data;
9538
9539                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
9540                 if (reg < 0)
9541                         return;
9542                 MLX5_ASSERT(reg != REG_NON);
9543                 if (reg == REG_C_0) {
9544                         struct mlx5_priv *priv = dev->data->dev_private;
9545                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9546                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9547
9548                         mask &= msk_c0;
9549                         mask <<= shl_c0;
9550                         value <<= shl_c0;
9551                 }
9552                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9553         }
9554 }
9555
9556 /**
9557  * Add vport metadata Reg C0 item to matcher
9558  *
9559  * @param[in, out] matcher
9560  *   Flow matcher.
9561  * @param[in, out] key
9562  *   Flow matcher value.
9563  * @param[in] reg
9564  *   Flow pattern to translate.
9565  */
9566 static void
9567 flow_dv_translate_item_meta_vport(void *matcher, void *key,
9568                                   uint32_t value, uint32_t mask)
9569 {
9570         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
9571 }
9572
9573 /**
9574  * Add tag item to matcher
9575  *
9576  * @param[in] dev
9577  *   The devich to configure through.
9578  * @param[in, out] matcher
9579  *   Flow matcher.
9580  * @param[in, out] key
9581  *   Flow matcher value.
9582  * @param[in] item
9583  *   Flow pattern to translate.
9584  */
9585 static void
9586 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
9587                                 void *matcher, void *key,
9588                                 const struct rte_flow_item *item)
9589 {
9590         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
9591         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
9592         uint32_t mask, value;
9593
9594         MLX5_ASSERT(tag_v);
9595         value = tag_v->data;
9596         mask = tag_m ? tag_m->data : UINT32_MAX;
9597         if (tag_v->id == REG_C_0) {
9598                 struct mlx5_priv *priv = dev->data->dev_private;
9599                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9600                 uint32_t shl_c0 = rte_bsf32(msk_c0);
9601
9602                 mask &= msk_c0;
9603                 mask <<= shl_c0;
9604                 value <<= shl_c0;
9605         }
9606         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
9607 }
9608
9609 /**
9610  * Add TAG item to matcher
9611  *
9612  * @param[in] dev
9613  *   The devich to configure through.
9614  * @param[in, out] matcher
9615  *   Flow matcher.
9616  * @param[in, out] key
9617  *   Flow matcher value.
9618  * @param[in] item
9619  *   Flow pattern to translate.
9620  */
9621 static void
9622 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
9623                            void *matcher, void *key,
9624                            const struct rte_flow_item *item)
9625 {
9626         const struct rte_flow_item_tag *tag_v = item->spec;
9627         const struct rte_flow_item_tag *tag_m = item->mask;
9628         enum modify_reg reg;
9629
9630         MLX5_ASSERT(tag_v);
9631         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
9632         /* Get the metadata register index for the tag. */
9633         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
9634         MLX5_ASSERT(reg > 0);
9635         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
9636 }
9637
9638 /**
9639  * Add source vport match to the specified matcher.
9640  *
9641  * @param[in, out] matcher
9642  *   Flow matcher.
9643  * @param[in, out] key
9644  *   Flow matcher value.
9645  * @param[in] port
9646  *   Source vport value to match
9647  * @param[in] mask
9648  *   Mask
9649  */
9650 static void
9651 flow_dv_translate_item_source_vport(void *matcher, void *key,
9652                                     int16_t port, uint16_t mask)
9653 {
9654         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9655         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9656
9657         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
9658         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
9659 }
9660
9661 /**
9662  * Translate port-id item to eswitch match on  port-id.
9663  *
9664  * @param[in] dev
9665  *   The devich to configure through.
9666  * @param[in, out] matcher
9667  *   Flow matcher.
9668  * @param[in, out] key
9669  *   Flow matcher value.
9670  * @param[in] item
9671  *   Flow pattern to translate.
9672  * @param[in]
9673  *   Flow attributes.
9674  *
9675  * @return
9676  *   0 on success, a negative errno value otherwise.
9677  */
9678 static int
9679 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
9680                                void *key, const struct rte_flow_item *item,
9681                                const struct rte_flow_attr *attr)
9682 {
9683         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
9684         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
9685         struct mlx5_priv *priv;
9686         uint16_t mask, id;
9687
9688         mask = pid_m ? pid_m->id : 0xffff;
9689         id = pid_v ? pid_v->id : dev->data->port_id;
9690         priv = mlx5_port_to_eswitch_info(id, item == NULL);
9691         if (!priv)
9692                 return -rte_errno;
9693         /*
9694          * Translate to vport field or to metadata, depending on mode.
9695          * Kernel can use either misc.source_port or half of C0 metadata
9696          * register.
9697          */
9698         if (priv->vport_meta_mask) {
9699                 /*
9700                  * Provide the hint for SW steering library
9701                  * to insert the flow into ingress domain and
9702                  * save the extra vport match.
9703                  */
9704                 if (mask == 0xffff && priv->vport_id == 0xffff &&
9705                     priv->pf_bond < 0 && attr->transfer)
9706                         flow_dv_translate_item_source_vport
9707                                 (matcher, key, priv->vport_id, mask);
9708                 /*
9709                  * We should always set the vport metadata register,
9710                  * otherwise the SW steering library can drop
9711                  * the rule if wire vport metadata value is not zero,
9712                  * it depends on kernel configuration.
9713                  */
9714                 flow_dv_translate_item_meta_vport(matcher, key,
9715                                                   priv->vport_meta_tag,
9716                                                   priv->vport_meta_mask);
9717         } else {
9718                 flow_dv_translate_item_source_vport(matcher, key,
9719                                                     priv->vport_id, mask);
9720         }
9721         return 0;
9722 }
9723
9724 /**
9725  * Add ICMP6 item to matcher and to the value.
9726  *
9727  * @param[in, out] matcher
9728  *   Flow matcher.
9729  * @param[in, out] key
9730  *   Flow matcher value.
9731  * @param[in] item
9732  *   Flow pattern to translate.
9733  * @param[in] inner
9734  *   Item is inner pattern.
9735  */
9736 static void
9737 flow_dv_translate_item_icmp6(void *matcher, void *key,
9738                               const struct rte_flow_item *item,
9739                               int inner)
9740 {
9741         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
9742         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
9743         void *headers_m;
9744         void *headers_v;
9745         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9746                                      misc_parameters_3);
9747         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9748         if (inner) {
9749                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9750                                          inner_headers);
9751                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9752         } else {
9753                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9754                                          outer_headers);
9755                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9756         }
9757         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9758         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
9759         if (!icmp6_v)
9760                 return;
9761         if (!icmp6_m)
9762                 icmp6_m = &rte_flow_item_icmp6_mask;
9763         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
9764         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
9765                  icmp6_v->type & icmp6_m->type);
9766         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
9767         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
9768                  icmp6_v->code & icmp6_m->code);
9769 }
9770
9771 /**
9772  * Add ICMP item to matcher and to the value.
9773  *
9774  * @param[in, out] matcher
9775  *   Flow matcher.
9776  * @param[in, out] key
9777  *   Flow matcher value.
9778  * @param[in] item
9779  *   Flow pattern to translate.
9780  * @param[in] inner
9781  *   Item is inner pattern.
9782  */
9783 static void
9784 flow_dv_translate_item_icmp(void *matcher, void *key,
9785                             const struct rte_flow_item *item,
9786                             int inner)
9787 {
9788         const struct rte_flow_item_icmp *icmp_m = item->mask;
9789         const struct rte_flow_item_icmp *icmp_v = item->spec;
9790         uint32_t icmp_header_data_m = 0;
9791         uint32_t icmp_header_data_v = 0;
9792         void *headers_m;
9793         void *headers_v;
9794         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9795                                      misc_parameters_3);
9796         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9797         if (inner) {
9798                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9799                                          inner_headers);
9800                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9801         } else {
9802                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9803                                          outer_headers);
9804                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9805         }
9806         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9807         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
9808         if (!icmp_v)
9809                 return;
9810         if (!icmp_m)
9811                 icmp_m = &rte_flow_item_icmp_mask;
9812         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
9813                  icmp_m->hdr.icmp_type);
9814         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
9815                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
9816         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
9817                  icmp_m->hdr.icmp_code);
9818         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
9819                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
9820         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
9821         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
9822         if (icmp_header_data_m) {
9823                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
9824                 icmp_header_data_v |=
9825                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
9826                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
9827                          icmp_header_data_m);
9828                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
9829                          icmp_header_data_v & icmp_header_data_m);
9830         }
9831 }
9832
9833 /**
9834  * Add GTP item to matcher and to the value.
9835  *
9836  * @param[in, out] matcher
9837  *   Flow matcher.
9838  * @param[in, out] key
9839  *   Flow matcher value.
9840  * @param[in] item
9841  *   Flow pattern to translate.
9842  * @param[in] inner
9843  *   Item is inner pattern.
9844  */
9845 static void
9846 flow_dv_translate_item_gtp(void *matcher, void *key,
9847                            const struct rte_flow_item *item, int inner)
9848 {
9849         const struct rte_flow_item_gtp *gtp_m = item->mask;
9850         const struct rte_flow_item_gtp *gtp_v = item->spec;
9851         void *headers_m;
9852         void *headers_v;
9853         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9854                                      misc_parameters_3);
9855         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9856         uint16_t dport = RTE_GTPU_UDP_PORT;
9857
9858         if (inner) {
9859                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9860                                          inner_headers);
9861                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9862         } else {
9863                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9864                                          outer_headers);
9865                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9866         }
9867         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9868                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9869                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9870         }
9871         if (!gtp_v)
9872                 return;
9873         if (!gtp_m)
9874                 gtp_m = &rte_flow_item_gtp_mask;
9875         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
9876                  gtp_m->v_pt_rsv_flags);
9877         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
9878                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
9879         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
9880         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
9881                  gtp_v->msg_type & gtp_m->msg_type);
9882         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
9883                  rte_be_to_cpu_32(gtp_m->teid));
9884         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
9885                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
9886 }
9887
9888 /**
9889  * Add GTP PSC item to matcher.
9890  *
9891  * @param[in, out] matcher
9892  *   Flow matcher.
9893  * @param[in, out] key
9894  *   Flow matcher value.
9895  * @param[in] item
9896  *   Flow pattern to translate.
9897  */
9898 static int
9899 flow_dv_translate_item_gtp_psc(void *matcher, void *key,
9900                                const struct rte_flow_item *item)
9901 {
9902         const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask;
9903         const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec;
9904         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9905                         misc_parameters_3);
9906         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9907         union {
9908                 uint32_t w32;
9909                 struct {
9910                         uint16_t seq_num;
9911                         uint8_t npdu_num;
9912                         uint8_t next_ext_header_type;
9913                 };
9914         } dw_2;
9915         uint8_t gtp_flags;
9916
9917         /* Always set E-flag match on one, regardless of GTP item settings. */
9918         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags);
9919         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9920         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags);
9921         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags);
9922         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9923         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags);
9924         /*Set next extension header type. */
9925         dw_2.seq_num = 0;
9926         dw_2.npdu_num = 0;
9927         dw_2.next_ext_header_type = 0xff;
9928         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2,
9929                  rte_cpu_to_be_32(dw_2.w32));
9930         dw_2.seq_num = 0;
9931         dw_2.npdu_num = 0;
9932         dw_2.next_ext_header_type = 0x85;
9933         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2,
9934                  rte_cpu_to_be_32(dw_2.w32));
9935         if (gtp_psc_v) {
9936                 union {
9937                         uint32_t w32;
9938                         struct {
9939                                 uint8_t len;
9940                                 uint8_t type_flags;
9941                                 uint8_t qfi;
9942                                 uint8_t reserved;
9943                         };
9944                 } dw_0;
9945
9946                 /*Set extension header PDU type and Qos. */
9947                 if (!gtp_psc_m)
9948                         gtp_psc_m = &rte_flow_item_gtp_psc_mask;
9949                 dw_0.w32 = 0;
9950                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->hdr.type);
9951                 dw_0.qfi = gtp_psc_m->hdr.qfi;
9952                 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0,
9953                          rte_cpu_to_be_32(dw_0.w32));
9954                 dw_0.w32 = 0;
9955                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->hdr.type &
9956                                                         gtp_psc_m->hdr.type);
9957                 dw_0.qfi = gtp_psc_v->hdr.qfi & gtp_psc_m->hdr.qfi;
9958                 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0,
9959                          rte_cpu_to_be_32(dw_0.w32));
9960         }
9961         return 0;
9962 }
9963
9964 /**
9965  * Add eCPRI item to matcher and to the value.
9966  *
9967  * @param[in] dev
9968  *   The devich to configure through.
9969  * @param[in, out] matcher
9970  *   Flow matcher.
9971  * @param[in, out] key
9972  *   Flow matcher value.
9973  * @param[in] item
9974  *   Flow pattern to translate.
9975  * @param[in] last_item
9976  *   Last item flags.
9977  */
9978 static void
9979 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
9980                              void *key, const struct rte_flow_item *item,
9981                              uint64_t last_item)
9982 {
9983         struct mlx5_priv *priv = dev->data->dev_private;
9984         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
9985         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
9986         struct rte_ecpri_common_hdr common;
9987         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
9988                                      misc_parameters_4);
9989         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
9990         uint32_t *samples;
9991         void *dw_m;
9992         void *dw_v;
9993
9994         /*
9995          * In case of eCPRI over Ethernet, if EtherType is not specified,
9996          * match on eCPRI EtherType implicitly.
9997          */
9998         if (last_item & MLX5_FLOW_LAYER_OUTER_L2) {
9999                 void *hdrs_m, *hdrs_v, *l2m, *l2v;
10000
10001                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
10002                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
10003                 l2m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, ethertype);
10004                 l2v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
10005                 if (*(uint16_t *)l2m == 0 && *(uint16_t *)l2v == 0) {
10006                         *(uint16_t *)l2m = UINT16_MAX;
10007                         *(uint16_t *)l2v = RTE_BE16(RTE_ETHER_TYPE_ECPRI);
10008                 }
10009         }
10010         if (!ecpri_v)
10011                 return;
10012         if (!ecpri_m)
10013                 ecpri_m = &rte_flow_item_ecpri_mask;
10014         /*
10015          * Maximal four DW samples are supported in a single matching now.
10016          * Two are used now for a eCPRI matching:
10017          * 1. Type: one byte, mask should be 0x00ff0000 in network order
10018          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
10019          *    if any.
10020          */
10021         if (!ecpri_m->hdr.common.u32)
10022                 return;
10023         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
10024         /* Need to take the whole DW as the mask to fill the entry. */
10025         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
10026                             prog_sample_field_value_0);
10027         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
10028                             prog_sample_field_value_0);
10029         /* Already big endian (network order) in the header. */
10030         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
10031         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32;
10032         /* Sample#0, used for matching type, offset 0. */
10033         MLX5_SET(fte_match_set_misc4, misc4_m,
10034                  prog_sample_field_id_0, samples[0]);
10035         /* It makes no sense to set the sample ID in the mask field. */
10036         MLX5_SET(fte_match_set_misc4, misc4_v,
10037                  prog_sample_field_id_0, samples[0]);
10038         /*
10039          * Checking if message body part needs to be matched.
10040          * Some wildcard rules only matching type field should be supported.
10041          */
10042         if (ecpri_m->hdr.dummy[0]) {
10043                 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32);
10044                 switch (common.type) {
10045                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
10046                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
10047                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
10048                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
10049                                             prog_sample_field_value_1);
10050                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
10051                                             prog_sample_field_value_1);
10052                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
10053                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] &
10054                                             ecpri_m->hdr.dummy[0];
10055                         /* Sample#1, to match message body, offset 4. */
10056                         MLX5_SET(fte_match_set_misc4, misc4_m,
10057                                  prog_sample_field_id_1, samples[1]);
10058                         MLX5_SET(fte_match_set_misc4, misc4_v,
10059                                  prog_sample_field_id_1, samples[1]);
10060                         break;
10061                 default:
10062                         /* Others, do not match any sample ID. */
10063                         break;
10064                 }
10065         }
10066 }
10067
10068 /*
10069  * Add connection tracking status item to matcher
10070  *
10071  * @param[in] dev
10072  *   The devich to configure through.
10073  * @param[in, out] matcher
10074  *   Flow matcher.
10075  * @param[in, out] key
10076  *   Flow matcher value.
10077  * @param[in] item
10078  *   Flow pattern to translate.
10079  */
10080 static void
10081 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev,
10082                               void *matcher, void *key,
10083                               const struct rte_flow_item *item)
10084 {
10085         uint32_t reg_value = 0;
10086         int reg_id;
10087         /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */
10088         uint32_t reg_mask = 0;
10089         const struct rte_flow_item_conntrack *spec = item->spec;
10090         const struct rte_flow_item_conntrack *mask = item->mask;
10091         uint32_t flags;
10092         struct rte_flow_error error;
10093
10094         if (!mask)
10095                 mask = &rte_flow_item_conntrack_mask;
10096         if (!spec || !mask->flags)
10097                 return;
10098         flags = spec->flags & mask->flags;
10099         /* The conflict should be checked in the validation. */
10100         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID)
10101                 reg_value |= MLX5_CT_SYNDROME_VALID;
10102         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10103                 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE;
10104         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID)
10105                 reg_value |= MLX5_CT_SYNDROME_INVALID;
10106         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)
10107                 reg_value |= MLX5_CT_SYNDROME_TRAP;
10108         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10109                 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET;
10110         if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID |
10111                            RTE_FLOW_CONNTRACK_PKT_STATE_INVALID |
10112                            RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))
10113                 reg_mask |= 0xc0;
10114         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10115                 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE;
10116         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10117                 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET;
10118         /* The REG_C_x value could be saved during startup. */
10119         reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error);
10120         if (reg_id == REG_NON)
10121                 return;
10122         flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id,
10123                                reg_value, reg_mask);
10124 }
10125
10126 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
10127
10128 #define HEADER_IS_ZERO(match_criteria, headers)                              \
10129         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
10130                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
10131
10132 /**
10133  * Calculate flow matcher enable bitmap.
10134  *
10135  * @param match_criteria
10136  *   Pointer to flow matcher criteria.
10137  *
10138  * @return
10139  *   Bitmap of enabled fields.
10140  */
10141 static uint8_t
10142 flow_dv_matcher_enable(uint32_t *match_criteria)
10143 {
10144         uint8_t match_criteria_enable;
10145
10146         match_criteria_enable =
10147                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
10148                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
10149         match_criteria_enable |=
10150                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
10151                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
10152         match_criteria_enable |=
10153                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
10154                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
10155         match_criteria_enable |=
10156                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
10157                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
10158         match_criteria_enable |=
10159                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
10160                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
10161         match_criteria_enable |=
10162                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
10163                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
10164         match_criteria_enable |=
10165                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_5)) <<
10166                 MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT;
10167         return match_criteria_enable;
10168 }
10169
10170 static void
10171 __flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria)
10172 {
10173         /*
10174          * Check flow matching criteria first, subtract misc5/4 length if flow
10175          * doesn't own misc5/4 parameters. In some old rdma-core releases,
10176          * misc5/4 are not supported, and matcher creation failure is expected
10177          * w/o subtration. If misc5 is provided, misc4 must be counted in since
10178          * misc5 is right after misc4.
10179          */
10180         if (!(match_criteria & (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT))) {
10181                 *size = MLX5_ST_SZ_BYTES(fte_match_param) -
10182                         MLX5_ST_SZ_BYTES(fte_match_set_misc5);
10183                 if (!(match_criteria & (1 <<
10184                         MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT))) {
10185                         *size -= MLX5_ST_SZ_BYTES(fte_match_set_misc4);
10186                 }
10187         }
10188 }
10189
10190 static struct mlx5_list_entry *
10191 flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
10192                          struct mlx5_list_entry *entry, void *cb_ctx)
10193 {
10194         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10195         struct mlx5_flow_dv_matcher *ref = ctx->data;
10196         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10197                                                             typeof(*tbl), tbl);
10198         struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
10199                                                             sizeof(*resource),
10200                                                             0, SOCKET_ID_ANY);
10201
10202         if (!resource) {
10203                 rte_flow_error_set(ctx->error, ENOMEM,
10204                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10205                                    "cannot create matcher");
10206                 return NULL;
10207         }
10208         memcpy(resource, entry, sizeof(*resource));
10209         resource->tbl = &tbl->tbl;
10210         return &resource->entry;
10211 }
10212
10213 static void
10214 flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
10215                              struct mlx5_list_entry *entry)
10216 {
10217         mlx5_free(entry);
10218 }
10219
10220 struct mlx5_list_entry *
10221 flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
10222 {
10223         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10224         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10225         struct rte_eth_dev *dev = ctx->dev;
10226         struct mlx5_flow_tbl_data_entry *tbl_data;
10227         struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
10228         struct rte_flow_error *error = ctx->error;
10229         union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
10230         struct mlx5_flow_tbl_resource *tbl;
10231         void *domain;
10232         uint32_t idx = 0;
10233         int ret;
10234
10235         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10236         if (!tbl_data) {
10237                 rte_flow_error_set(error, ENOMEM,
10238                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10239                                    NULL,
10240                                    "cannot allocate flow table data entry");
10241                 return NULL;
10242         }
10243         tbl_data->idx = idx;
10244         tbl_data->tunnel = tt_prm->tunnel;
10245         tbl_data->group_id = tt_prm->group_id;
10246         tbl_data->external = !!tt_prm->external;
10247         tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
10248         tbl_data->is_egress = !!key.is_egress;
10249         tbl_data->is_transfer = !!key.is_fdb;
10250         tbl_data->dummy = !!key.dummy;
10251         tbl_data->level = key.level;
10252         tbl_data->id = key.id;
10253         tbl = &tbl_data->tbl;
10254         if (key.dummy)
10255                 return &tbl_data->entry;
10256         if (key.is_fdb)
10257                 domain = sh->fdb_domain;
10258         else if (key.is_egress)
10259                 domain = sh->tx_domain;
10260         else
10261                 domain = sh->rx_domain;
10262         ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
10263         if (ret) {
10264                 rte_flow_error_set(error, ENOMEM,
10265                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10266                                    NULL, "cannot create flow table object");
10267                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10268                 return NULL;
10269         }
10270         if (key.level != 0) {
10271                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
10272                                         (tbl->obj, &tbl_data->jump.action);
10273                 if (ret) {
10274                         rte_flow_error_set(error, ENOMEM,
10275                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10276                                            NULL,
10277                                            "cannot create flow jump action");
10278                         mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10279                         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10280                         return NULL;
10281                 }
10282         }
10283         MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
10284               key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
10285               key.level, key.id);
10286         tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
10287                                               flow_dv_matcher_create_cb,
10288                                               flow_dv_matcher_match_cb,
10289                                               flow_dv_matcher_remove_cb,
10290                                               flow_dv_matcher_clone_cb,
10291                                               flow_dv_matcher_clone_free_cb);
10292         if (!tbl_data->matchers) {
10293                 rte_flow_error_set(error, ENOMEM,
10294                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10295                                    NULL,
10296                                    "cannot create tbl matcher list");
10297                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10298                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10299                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10300                 return NULL;
10301         }
10302         return &tbl_data->entry;
10303 }
10304
10305 int
10306 flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10307                      void *cb_ctx)
10308 {
10309         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10310         struct mlx5_flow_tbl_data_entry *tbl_data =
10311                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10312         union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
10313
10314         return tbl_data->level != key.level ||
10315                tbl_data->id != key.id ||
10316                tbl_data->dummy != key.dummy ||
10317                tbl_data->is_transfer != !!key.is_fdb ||
10318                tbl_data->is_egress != !!key.is_egress;
10319 }
10320
10321 struct mlx5_list_entry *
10322 flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10323                       void *cb_ctx)
10324 {
10325         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10326         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10327         struct mlx5_flow_tbl_data_entry *tbl_data;
10328         struct rte_flow_error *error = ctx->error;
10329         uint32_t idx = 0;
10330
10331         tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10332         if (!tbl_data) {
10333                 rte_flow_error_set(error, ENOMEM,
10334                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10335                                    NULL,
10336                                    "cannot allocate flow table data entry");
10337                 return NULL;
10338         }
10339         memcpy(tbl_data, oentry, sizeof(*tbl_data));
10340         tbl_data->idx = idx;
10341         return &tbl_data->entry;
10342 }
10343
10344 void
10345 flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10346 {
10347         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10348         struct mlx5_flow_tbl_data_entry *tbl_data =
10349                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10350
10351         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10352 }
10353
10354 /**
10355  * Get a flow table.
10356  *
10357  * @param[in, out] dev
10358  *   Pointer to rte_eth_dev structure.
10359  * @param[in] table_level
10360  *   Table level to use.
10361  * @param[in] egress
10362  *   Direction of the table.
10363  * @param[in] transfer
10364  *   E-Switch or NIC flow.
10365  * @param[in] dummy
10366  *   Dummy entry for dv API.
10367  * @param[in] table_id
10368  *   Table id to use.
10369  * @param[out] error
10370  *   pointer to error structure.
10371  *
10372  * @return
10373  *   Returns tables resource based on the index, NULL in case of failed.
10374  */
10375 struct mlx5_flow_tbl_resource *
10376 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
10377                          uint32_t table_level, uint8_t egress,
10378                          uint8_t transfer,
10379                          bool external,
10380                          const struct mlx5_flow_tunnel *tunnel,
10381                          uint32_t group_id, uint8_t dummy,
10382                          uint32_t table_id,
10383                          struct rte_flow_error *error)
10384 {
10385         struct mlx5_priv *priv = dev->data->dev_private;
10386         union mlx5_flow_tbl_key table_key = {
10387                 {
10388                         .level = table_level,
10389                         .id = table_id,
10390                         .reserved = 0,
10391                         .dummy = !!dummy,
10392                         .is_fdb = !!transfer,
10393                         .is_egress = !!egress,
10394                 }
10395         };
10396         struct mlx5_flow_tbl_tunnel_prm tt_prm = {
10397                 .tunnel = tunnel,
10398                 .group_id = group_id,
10399                 .external = external,
10400         };
10401         struct mlx5_flow_cb_ctx ctx = {
10402                 .dev = dev,
10403                 .error = error,
10404                 .data = &table_key.v64,
10405                 .data2 = &tt_prm,
10406         };
10407         struct mlx5_list_entry *entry;
10408         struct mlx5_flow_tbl_data_entry *tbl_data;
10409
10410         entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
10411         if (!entry) {
10412                 rte_flow_error_set(error, ENOMEM,
10413                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10414                                    "cannot get table");
10415                 return NULL;
10416         }
10417         DRV_LOG(DEBUG, "table_level %u table_id %u "
10418                 "tunnel %u group %u registered.",
10419                 table_level, table_id,
10420                 tunnel ? tunnel->tunnel_id : 0, group_id);
10421         tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10422         return &tbl_data->tbl;
10423 }
10424
10425 void
10426 flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10427 {
10428         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10429         struct mlx5_flow_tbl_data_entry *tbl_data =
10430                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10431
10432         MLX5_ASSERT(entry && sh);
10433         if (tbl_data->jump.action)
10434                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10435         if (tbl_data->tbl.obj)
10436                 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
10437         if (tbl_data->tunnel_offload && tbl_data->external) {
10438                 struct mlx5_list_entry *he;
10439                 struct mlx5_hlist *tunnel_grp_hash;
10440                 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10441                 union tunnel_tbl_key tunnel_key = {
10442                         .tunnel_id = tbl_data->tunnel ?
10443                                         tbl_data->tunnel->tunnel_id : 0,
10444                         .group = tbl_data->group_id
10445                 };
10446                 uint32_t table_level = tbl_data->level;
10447                 struct mlx5_flow_cb_ctx ctx = {
10448                         .data = (void *)&tunnel_key.val,
10449                 };
10450
10451                 tunnel_grp_hash = tbl_data->tunnel ?
10452                                         tbl_data->tunnel->groups :
10453                                         thub->groups;
10454                 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
10455                 if (he)
10456                         mlx5_hlist_unregister(tunnel_grp_hash, he);
10457                 DRV_LOG(DEBUG,
10458                         "table_level %u id %u tunnel %u group %u released.",
10459                         table_level,
10460                         tbl_data->id,
10461                         tbl_data->tunnel ?
10462                         tbl_data->tunnel->tunnel_id : 0,
10463                         tbl_data->group_id);
10464         }
10465         mlx5_list_destroy(tbl_data->matchers);
10466         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10467 }
10468
10469 /**
10470  * Release a flow table.
10471  *
10472  * @param[in] sh
10473  *   Pointer to device shared structure.
10474  * @param[in] tbl
10475  *   Table resource to be released.
10476  *
10477  * @return
10478  *   Returns 0 if table was released, else return 1;
10479  */
10480 static int
10481 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
10482                              struct mlx5_flow_tbl_resource *tbl)
10483 {
10484         struct mlx5_flow_tbl_data_entry *tbl_data =
10485                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10486
10487         if (!tbl)
10488                 return 0;
10489         return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
10490 }
10491
10492 int
10493 flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
10494                          struct mlx5_list_entry *entry, void *cb_ctx)
10495 {
10496         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10497         struct mlx5_flow_dv_matcher *ref = ctx->data;
10498         struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur),
10499                                                         entry);
10500
10501         return cur->crc != ref->crc ||
10502                cur->priority != ref->priority ||
10503                memcmp((const void *)cur->mask.buf,
10504                       (const void *)ref->mask.buf, ref->mask.size);
10505 }
10506
10507 struct mlx5_list_entry *
10508 flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
10509 {
10510         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10511         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10512         struct mlx5_flow_dv_matcher *ref = ctx->data;
10513         struct mlx5_flow_dv_matcher *resource;
10514         struct mlx5dv_flow_matcher_attr dv_attr = {
10515                 .type = IBV_FLOW_ATTR_NORMAL,
10516                 .match_mask = (void *)&ref->mask,
10517         };
10518         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10519                                                             typeof(*tbl), tbl);
10520         int ret;
10521
10522         resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
10523                                SOCKET_ID_ANY);
10524         if (!resource) {
10525                 rte_flow_error_set(ctx->error, ENOMEM,
10526                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10527                                    "cannot create matcher");
10528                 return NULL;
10529         }
10530         *resource = *ref;
10531         dv_attr.match_criteria_enable =
10532                 flow_dv_matcher_enable(resource->mask.buf);
10533         __flow_dv_adjust_buf_size(&ref->mask.size,
10534                                   dv_attr.match_criteria_enable);
10535         dv_attr.priority = ref->priority;
10536         if (tbl->is_egress)
10537                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
10538         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
10539                                                &resource->matcher_object);
10540         if (ret) {
10541                 mlx5_free(resource);
10542                 rte_flow_error_set(ctx->error, ENOMEM,
10543                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10544                                    "cannot create matcher");
10545                 return NULL;
10546         }
10547         return &resource->entry;
10548 }
10549
10550 /**
10551  * Register the flow matcher.
10552  *
10553  * @param[in, out] dev
10554  *   Pointer to rte_eth_dev structure.
10555  * @param[in, out] matcher
10556  *   Pointer to flow matcher.
10557  * @param[in, out] key
10558  *   Pointer to flow table key.
10559  * @parm[in, out] dev_flow
10560  *   Pointer to the dev_flow.
10561  * @param[out] error
10562  *   pointer to error structure.
10563  *
10564  * @return
10565  *   0 on success otherwise -errno and errno is set.
10566  */
10567 static int
10568 flow_dv_matcher_register(struct rte_eth_dev *dev,
10569                          struct mlx5_flow_dv_matcher *ref,
10570                          union mlx5_flow_tbl_key *key,
10571                          struct mlx5_flow *dev_flow,
10572                          const struct mlx5_flow_tunnel *tunnel,
10573                          uint32_t group_id,
10574                          struct rte_flow_error *error)
10575 {
10576         struct mlx5_list_entry *entry;
10577         struct mlx5_flow_dv_matcher *resource;
10578         struct mlx5_flow_tbl_resource *tbl;
10579         struct mlx5_flow_tbl_data_entry *tbl_data;
10580         struct mlx5_flow_cb_ctx ctx = {
10581                 .error = error,
10582                 .data = ref,
10583         };
10584         /**
10585          * tunnel offload API requires this registration for cases when
10586          * tunnel match rule was inserted before tunnel set rule.
10587          */
10588         tbl = flow_dv_tbl_resource_get(dev, key->level,
10589                                        key->is_egress, key->is_fdb,
10590                                        dev_flow->external, tunnel,
10591                                        group_id, 0, key->id, error);
10592         if (!tbl)
10593                 return -rte_errno;      /* No need to refill the error info */
10594         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10595         ref->tbl = tbl;
10596         entry = mlx5_list_register(tbl_data->matchers, &ctx);
10597         if (!entry) {
10598                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
10599                 return rte_flow_error_set(error, ENOMEM,
10600                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10601                                           "cannot allocate ref memory");
10602         }
10603         resource = container_of(entry, typeof(*resource), entry);
10604         dev_flow->handle->dvh.matcher = resource;
10605         return 0;
10606 }
10607
10608 struct mlx5_list_entry *
10609 flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
10610 {
10611         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10612         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10613         struct mlx5_flow_dv_tag_resource *entry;
10614         uint32_t idx = 0;
10615         int ret;
10616
10617         entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10618         if (!entry) {
10619                 rte_flow_error_set(ctx->error, ENOMEM,
10620                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10621                                    "cannot allocate resource memory");
10622                 return NULL;
10623         }
10624         entry->idx = idx;
10625         entry->tag_id = *(uint32_t *)(ctx->data);
10626         ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
10627                                                   &entry->action);
10628         if (ret) {
10629                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
10630                 rte_flow_error_set(ctx->error, ENOMEM,
10631                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10632                                    NULL, "cannot create action");
10633                 return NULL;
10634         }
10635         return &entry->entry;
10636 }
10637
10638 int
10639 flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10640                      void *cb_ctx)
10641 {
10642         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10643         struct mlx5_flow_dv_tag_resource *tag =
10644                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10645
10646         return *(uint32_t *)(ctx->data) != tag->tag_id;
10647 }
10648
10649 struct mlx5_list_entry *
10650 flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10651                      void *cb_ctx)
10652 {
10653         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10654         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10655         struct mlx5_flow_dv_tag_resource *entry;
10656         uint32_t idx = 0;
10657
10658         entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10659         if (!entry) {
10660                 rte_flow_error_set(ctx->error, ENOMEM,
10661                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10662                                    "cannot allocate tag resource memory");
10663                 return NULL;
10664         }
10665         memcpy(entry, oentry, sizeof(*entry));
10666         entry->idx = idx;
10667         return &entry->entry;
10668 }
10669
10670 void
10671 flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10672 {
10673         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10674         struct mlx5_flow_dv_tag_resource *tag =
10675                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10676
10677         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10678 }
10679
10680 /**
10681  * Find existing tag resource or create and register a new one.
10682  *
10683  * @param dev[in, out]
10684  *   Pointer to rte_eth_dev structure.
10685  * @param[in, out] tag_be24
10686  *   Tag value in big endian then R-shift 8.
10687  * @parm[in, out] dev_flow
10688  *   Pointer to the dev_flow.
10689  * @param[out] error
10690  *   pointer to error structure.
10691  *
10692  * @return
10693  *   0 on success otherwise -errno and errno is set.
10694  */
10695 static int
10696 flow_dv_tag_resource_register
10697                         (struct rte_eth_dev *dev,
10698                          uint32_t tag_be24,
10699                          struct mlx5_flow *dev_flow,
10700                          struct rte_flow_error *error)
10701 {
10702         struct mlx5_priv *priv = dev->data->dev_private;
10703         struct mlx5_flow_dv_tag_resource *resource;
10704         struct mlx5_list_entry *entry;
10705         struct mlx5_flow_cb_ctx ctx = {
10706                                         .error = error,
10707                                         .data = &tag_be24,
10708                                         };
10709         struct mlx5_hlist *tag_table;
10710
10711         tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
10712                                       "tags",
10713                                       MLX5_TAGS_HLIST_ARRAY_SIZE,
10714                                       false, false, priv->sh,
10715                                       flow_dv_tag_create_cb,
10716                                       flow_dv_tag_match_cb,
10717                                       flow_dv_tag_remove_cb,
10718                                       flow_dv_tag_clone_cb,
10719                                       flow_dv_tag_clone_free_cb);
10720         if (unlikely(!tag_table))
10721                 return -rte_errno;
10722         entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
10723         if (entry) {
10724                 resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
10725                                         entry);
10726                 dev_flow->handle->dvh.rix_tag = resource->idx;
10727                 dev_flow->dv.tag_resource = resource;
10728                 return 0;
10729         }
10730         return -rte_errno;
10731 }
10732
10733 void
10734 flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10735 {
10736         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10737         struct mlx5_flow_dv_tag_resource *tag =
10738                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10739
10740         MLX5_ASSERT(tag && sh && tag->action);
10741         claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
10742         DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag);
10743         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10744 }
10745
10746 /**
10747  * Release the tag.
10748  *
10749  * @param dev
10750  *   Pointer to Ethernet device.
10751  * @param tag_idx
10752  *   Tag index.
10753  *
10754  * @return
10755  *   1 while a reference on it exists, 0 when freed.
10756  */
10757 static int
10758 flow_dv_tag_release(struct rte_eth_dev *dev,
10759                     uint32_t tag_idx)
10760 {
10761         struct mlx5_priv *priv = dev->data->dev_private;
10762         struct mlx5_flow_dv_tag_resource *tag;
10763
10764         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
10765         if (!tag)
10766                 return 0;
10767         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
10768                 dev->data->port_id, (void *)tag, tag->entry.ref_cnt);
10769         return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry);
10770 }
10771
10772 /**
10773  * Translate port ID action to vport.
10774  *
10775  * @param[in] dev
10776  *   Pointer to rte_eth_dev structure.
10777  * @param[in] action
10778  *   Pointer to the port ID action.
10779  * @param[out] dst_port_id
10780  *   The target port ID.
10781  * @param[out] error
10782  *   Pointer to the error structure.
10783  *
10784  * @return
10785  *   0 on success, a negative errno value otherwise and rte_errno is set.
10786  */
10787 static int
10788 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
10789                                  const struct rte_flow_action *action,
10790                                  uint32_t *dst_port_id,
10791                                  struct rte_flow_error *error)
10792 {
10793         uint32_t port;
10794         struct mlx5_priv *priv;
10795         const struct rte_flow_action_port_id *conf =
10796                         (const struct rte_flow_action_port_id *)action->conf;
10797
10798         port = conf->original ? dev->data->port_id : conf->id;
10799         priv = mlx5_port_to_eswitch_info(port, false);
10800         if (!priv)
10801                 return rte_flow_error_set(error, -rte_errno,
10802                                           RTE_FLOW_ERROR_TYPE_ACTION,
10803                                           NULL,
10804                                           "No eswitch info was found for port");
10805 #ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT
10806         /*
10807          * This parameter is transferred to
10808          * mlx5dv_dr_action_create_dest_ib_port().
10809          */
10810         *dst_port_id = priv->dev_port;
10811 #else
10812         /*
10813          * Legacy mode, no LAG configurations is supported.
10814          * This parameter is transferred to
10815          * mlx5dv_dr_action_create_dest_vport().
10816          */
10817         *dst_port_id = priv->vport_id;
10818 #endif
10819         return 0;
10820 }
10821
10822 /**
10823  * Create a counter with aging configuration.
10824  *
10825  * @param[in] dev
10826  *   Pointer to rte_eth_dev structure.
10827  * @param[in] dev_flow
10828  *   Pointer to the mlx5_flow.
10829  * @param[out] count
10830  *   Pointer to the counter action configuration.
10831  * @param[in] age
10832  *   Pointer to the aging action configuration.
10833  *
10834  * @return
10835  *   Index to flow counter on success, 0 otherwise.
10836  */
10837 static uint32_t
10838 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
10839                                 struct mlx5_flow *dev_flow,
10840                                 const struct rte_flow_action_count *count,
10841                                 const struct rte_flow_action_age *age)
10842 {
10843         uint32_t counter;
10844         struct mlx5_age_param *age_param;
10845
10846         if (count && count->shared)
10847                 counter = flow_dv_counter_get_shared(dev, count->id);
10848         else
10849                 counter = flow_dv_counter_alloc(dev, !!age);
10850         if (!counter || age == NULL)
10851                 return counter;
10852         age_param = flow_dv_counter_idx_get_age(dev, counter);
10853         age_param->context = age->context ? age->context :
10854                 (void *)(uintptr_t)(dev_flow->flow_idx);
10855         age_param->timeout = age->timeout;
10856         age_param->port_id = dev->data->port_id;
10857         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
10858         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
10859         return counter;
10860 }
10861
10862 /**
10863  * Add Tx queue matcher
10864  *
10865  * @param[in] dev
10866  *   Pointer to the dev struct.
10867  * @param[in, out] matcher
10868  *   Flow matcher.
10869  * @param[in, out] key
10870  *   Flow matcher value.
10871  * @param[in] item
10872  *   Flow pattern to translate.
10873  * @param[in] inner
10874  *   Item is inner pattern.
10875  */
10876 static void
10877 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
10878                                 void *matcher, void *key,
10879                                 const struct rte_flow_item *item)
10880 {
10881         const struct mlx5_rte_flow_item_tx_queue *queue_m;
10882         const struct mlx5_rte_flow_item_tx_queue *queue_v;
10883         void *misc_m =
10884                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
10885         void *misc_v =
10886                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
10887         struct mlx5_txq_ctrl *txq;
10888         uint32_t queue;
10889
10890
10891         queue_m = (const void *)item->mask;
10892         if (!queue_m)
10893                 return;
10894         queue_v = (const void *)item->spec;
10895         if (!queue_v)
10896                 return;
10897         txq = mlx5_txq_get(dev, queue_v->queue);
10898         if (!txq)
10899                 return;
10900         queue = txq->obj->sq->id;
10901         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
10902         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
10903                  queue & queue_m->queue);
10904         mlx5_txq_release(dev, queue_v->queue);
10905 }
10906
10907 /**
10908  * Set the hash fields according to the @p flow information.
10909  *
10910  * @param[in] dev_flow
10911  *   Pointer to the mlx5_flow.
10912  * @param[in] rss_desc
10913  *   Pointer to the mlx5_flow_rss_desc.
10914  */
10915 static void
10916 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
10917                        struct mlx5_flow_rss_desc *rss_desc)
10918 {
10919         uint64_t items = dev_flow->handle->layers;
10920         int rss_inner = 0;
10921         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
10922
10923         dev_flow->hash_fields = 0;
10924 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
10925         if (rss_desc->level >= 2)
10926                 rss_inner = 1;
10927 #endif
10928         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
10929             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
10930                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
10931                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10932                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
10933                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10934                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
10935                         else
10936                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
10937                 }
10938         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
10939                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
10940                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
10941                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10942                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
10943                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10944                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
10945                         else
10946                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
10947                 }
10948         }
10949         if (dev_flow->hash_fields == 0)
10950                 /*
10951                  * There is no match between the RSS types and the
10952                  * L3 protocol (IPv4/IPv6) defined in the flow rule.
10953                  */
10954                 return;
10955         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
10956             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
10957                 if (rss_types & ETH_RSS_UDP) {
10958                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10959                                 dev_flow->hash_fields |=
10960                                                 IBV_RX_HASH_SRC_PORT_UDP;
10961                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10962                                 dev_flow->hash_fields |=
10963                                                 IBV_RX_HASH_DST_PORT_UDP;
10964                         else
10965                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
10966                 }
10967         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
10968                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
10969                 if (rss_types & ETH_RSS_TCP) {
10970                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10971                                 dev_flow->hash_fields |=
10972                                                 IBV_RX_HASH_SRC_PORT_TCP;
10973                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10974                                 dev_flow->hash_fields |=
10975                                                 IBV_RX_HASH_DST_PORT_TCP;
10976                         else
10977                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
10978                 }
10979         }
10980         if (rss_inner)
10981                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
10982 }
10983
10984 /**
10985  * Prepare an Rx Hash queue.
10986  *
10987  * @param dev
10988  *   Pointer to Ethernet device.
10989  * @param[in] dev_flow
10990  *   Pointer to the mlx5_flow.
10991  * @param[in] rss_desc
10992  *   Pointer to the mlx5_flow_rss_desc.
10993  * @param[out] hrxq_idx
10994  *   Hash Rx queue index.
10995  *
10996  * @return
10997  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
10998  */
10999 static struct mlx5_hrxq *
11000 flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
11001                      struct mlx5_flow *dev_flow,
11002                      struct mlx5_flow_rss_desc *rss_desc,
11003                      uint32_t *hrxq_idx)
11004 {
11005         struct mlx5_priv *priv = dev->data->dev_private;
11006         struct mlx5_flow_handle *dh = dev_flow->handle;
11007         struct mlx5_hrxq *hrxq;
11008
11009         MLX5_ASSERT(rss_desc->queue_num);
11010         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
11011         rss_desc->hash_fields = dev_flow->hash_fields;
11012         rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL);
11013         rss_desc->shared_rss = 0;
11014         if (rss_desc->hash_fields == 0)
11015                 rss_desc->queue_num = 1;
11016         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
11017         if (!*hrxq_idx)
11018                 return NULL;
11019         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
11020                               *hrxq_idx);
11021         return hrxq;
11022 }
11023
11024 /**
11025  * Release sample sub action resource.
11026  *
11027  * @param[in, out] dev
11028  *   Pointer to rte_eth_dev structure.
11029  * @param[in] act_res
11030  *   Pointer to sample sub action resource.
11031  */
11032 static void
11033 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
11034                                    struct mlx5_flow_sub_actions_idx *act_res)
11035 {
11036         if (act_res->rix_hrxq) {
11037                 mlx5_hrxq_release(dev, act_res->rix_hrxq);
11038                 act_res->rix_hrxq = 0;
11039         }
11040         if (act_res->rix_encap_decap) {
11041                 flow_dv_encap_decap_resource_release(dev,
11042                                                      act_res->rix_encap_decap);
11043                 act_res->rix_encap_decap = 0;
11044         }
11045         if (act_res->rix_port_id_action) {
11046                 flow_dv_port_id_action_resource_release(dev,
11047                                                 act_res->rix_port_id_action);
11048                 act_res->rix_port_id_action = 0;
11049         }
11050         if (act_res->rix_tag) {
11051                 flow_dv_tag_release(dev, act_res->rix_tag);
11052                 act_res->rix_tag = 0;
11053         }
11054         if (act_res->rix_jump) {
11055                 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump);
11056                 act_res->rix_jump = 0;
11057         }
11058 }
11059
11060 int
11061 flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
11062                         struct mlx5_list_entry *entry, void *cb_ctx)
11063 {
11064         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11065         struct rte_eth_dev *dev = ctx->dev;
11066         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11067         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
11068                                                               typeof(*resource),
11069                                                               entry);
11070
11071         if (ctx_resource->ratio == resource->ratio &&
11072             ctx_resource->ft_type == resource->ft_type &&
11073             ctx_resource->ft_id == resource->ft_id &&
11074             ctx_resource->set_action == resource->set_action &&
11075             !memcmp((void *)&ctx_resource->sample_act,
11076                     (void *)&resource->sample_act,
11077                     sizeof(struct mlx5_flow_sub_actions_list))) {
11078                 /*
11079                  * Existing sample action should release the prepared
11080                  * sub-actions reference counter.
11081                  */
11082                 flow_dv_sample_sub_actions_release(dev,
11083                                                    &ctx_resource->sample_idx);
11084                 return 0;
11085         }
11086         return 1;
11087 }
11088
11089 struct mlx5_list_entry *
11090 flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11091 {
11092         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11093         struct rte_eth_dev *dev = ctx->dev;
11094         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11095         void **sample_dv_actions = ctx_resource->sub_actions;
11096         struct mlx5_flow_dv_sample_resource *resource;
11097         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
11098         struct mlx5_priv *priv = dev->data->dev_private;
11099         struct mlx5_dev_ctx_shared *sh = priv->sh;
11100         struct mlx5_flow_tbl_resource *tbl;
11101         uint32_t idx = 0;
11102         const uint32_t next_ft_step = 1;
11103         uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
11104         uint8_t is_egress = 0;
11105         uint8_t is_transfer = 0;
11106         struct rte_flow_error *error = ctx->error;
11107
11108         /* Register new sample resource. */
11109         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11110         if (!resource) {
11111                 rte_flow_error_set(error, ENOMEM,
11112                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11113                                           NULL,
11114                                           "cannot allocate resource memory");
11115                 return NULL;
11116         }
11117         *resource = *ctx_resource;
11118         /* Create normal path table level */
11119         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11120                 is_transfer = 1;
11121         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
11122                 is_egress = 1;
11123         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
11124                                         is_egress, is_transfer,
11125                                         true, NULL, 0, 0, 0, error);
11126         if (!tbl) {
11127                 rte_flow_error_set(error, ENOMEM,
11128                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11129                                           NULL,
11130                                           "fail to create normal path table "
11131                                           "for sample");
11132                 goto error;
11133         }
11134         resource->normal_path_tbl = tbl;
11135         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
11136                 if (!sh->default_miss_action) {
11137                         rte_flow_error_set(error, ENOMEM,
11138                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11139                                                 NULL,
11140                                                 "default miss action was not "
11141                                                 "created");
11142                         goto error;
11143                 }
11144                 sample_dv_actions[ctx_resource->sample_act.actions_num++] =
11145                                                 sh->default_miss_action;
11146         }
11147         /* Create a DR sample action */
11148         sampler_attr.sample_ratio = resource->ratio;
11149         sampler_attr.default_next_table = tbl->obj;
11150         sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
11151         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
11152                                                         &sample_dv_actions[0];
11153         sampler_attr.action = resource->set_action;
11154         if (mlx5_os_flow_dr_create_flow_action_sampler
11155                         (&sampler_attr, &resource->verbs_action)) {
11156                 rte_flow_error_set(error, ENOMEM,
11157                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11158                                         NULL, "cannot create sample action");
11159                 goto error;
11160         }
11161         resource->idx = idx;
11162         resource->dev = dev;
11163         return &resource->entry;
11164 error:
11165         if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
11166                 flow_dv_sample_sub_actions_release(dev,
11167                                                    &resource->sample_idx);
11168         if (resource->normal_path_tbl)
11169                 flow_dv_tbl_resource_release(MLX5_SH(dev),
11170                                 resource->normal_path_tbl);
11171         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
11172         return NULL;
11173
11174 }
11175
11176 struct mlx5_list_entry *
11177 flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
11178                          struct mlx5_list_entry *entry __rte_unused,
11179                          void *cb_ctx)
11180 {
11181         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11182         struct rte_eth_dev *dev = ctx->dev;
11183         struct mlx5_flow_dv_sample_resource *resource;
11184         struct mlx5_priv *priv = dev->data->dev_private;
11185         struct mlx5_dev_ctx_shared *sh = priv->sh;
11186         uint32_t idx = 0;
11187
11188         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11189         if (!resource) {
11190                 rte_flow_error_set(ctx->error, ENOMEM,
11191                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11192                                           NULL,
11193                                           "cannot allocate resource memory");
11194                 return NULL;
11195         }
11196         memcpy(resource, entry, sizeof(*resource));
11197         resource->idx = idx;
11198         resource->dev = dev;
11199         return &resource->entry;
11200 }
11201
11202 void
11203 flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
11204                              struct mlx5_list_entry *entry)
11205 {
11206         struct mlx5_flow_dv_sample_resource *resource =
11207                                   container_of(entry, typeof(*resource), entry);
11208         struct rte_eth_dev *dev = resource->dev;
11209         struct mlx5_priv *priv = dev->data->dev_private;
11210
11211         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
11212 }
11213
11214 /**
11215  * Find existing sample resource or create and register a new one.
11216  *
11217  * @param[in, out] dev
11218  *   Pointer to rte_eth_dev structure.
11219  * @param[in] ref
11220  *   Pointer to sample resource reference.
11221  * @parm[in, out] dev_flow
11222  *   Pointer to the dev_flow.
11223  * @param[out] error
11224  *   pointer to error structure.
11225  *
11226  * @return
11227  *   0 on success otherwise -errno and errno is set.
11228  */
11229 static int
11230 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
11231                          struct mlx5_flow_dv_sample_resource *ref,
11232                          struct mlx5_flow *dev_flow,
11233                          struct rte_flow_error *error)
11234 {
11235         struct mlx5_flow_dv_sample_resource *resource;
11236         struct mlx5_list_entry *entry;
11237         struct mlx5_priv *priv = dev->data->dev_private;
11238         struct mlx5_flow_cb_ctx ctx = {
11239                 .dev = dev,
11240                 .error = error,
11241                 .data = ref,
11242         };
11243
11244         entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
11245         if (!entry)
11246                 return -rte_errno;
11247         resource = container_of(entry, typeof(*resource), entry);
11248         dev_flow->handle->dvh.rix_sample = resource->idx;
11249         dev_flow->dv.sample_res = resource;
11250         return 0;
11251 }
11252
11253 int
11254 flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
11255                             struct mlx5_list_entry *entry, void *cb_ctx)
11256 {
11257         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11258         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11259         struct rte_eth_dev *dev = ctx->dev;
11260         struct mlx5_flow_dv_dest_array_resource *resource =
11261                                   container_of(entry, typeof(*resource), entry);
11262         uint32_t idx = 0;
11263
11264         if (ctx_resource->num_of_dest == resource->num_of_dest &&
11265             ctx_resource->ft_type == resource->ft_type &&
11266             !memcmp((void *)resource->sample_act,
11267                     (void *)ctx_resource->sample_act,
11268                    (ctx_resource->num_of_dest *
11269                    sizeof(struct mlx5_flow_sub_actions_list)))) {
11270                 /*
11271                  * Existing sample action should release the prepared
11272                  * sub-actions reference counter.
11273                  */
11274                 for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11275                         flow_dv_sample_sub_actions_release(dev,
11276                                         &ctx_resource->sample_idx[idx]);
11277                 return 0;
11278         }
11279         return 1;
11280 }
11281
11282 struct mlx5_list_entry *
11283 flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11284 {
11285         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11286         struct rte_eth_dev *dev = ctx->dev;
11287         struct mlx5_flow_dv_dest_array_resource *resource;
11288         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11289         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
11290         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
11291         struct mlx5_priv *priv = dev->data->dev_private;
11292         struct mlx5_dev_ctx_shared *sh = priv->sh;
11293         struct mlx5_flow_sub_actions_list *sample_act;
11294         struct mlx5dv_dr_domain *domain;
11295         uint32_t idx = 0, res_idx = 0;
11296         struct rte_flow_error *error = ctx->error;
11297         uint64_t action_flags;
11298         int ret;
11299
11300         /* Register new destination array resource. */
11301         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11302                                             &res_idx);
11303         if (!resource) {
11304                 rte_flow_error_set(error, ENOMEM,
11305                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11306                                           NULL,
11307                                           "cannot allocate resource memory");
11308                 return NULL;
11309         }
11310         *resource = *ctx_resource;
11311         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11312                 domain = sh->fdb_domain;
11313         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
11314                 domain = sh->rx_domain;
11315         else
11316                 domain = sh->tx_domain;
11317         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11318                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
11319                                  mlx5_malloc(MLX5_MEM_ZERO,
11320                                  sizeof(struct mlx5dv_dr_action_dest_attr),
11321                                  0, SOCKET_ID_ANY);
11322                 if (!dest_attr[idx]) {
11323                         rte_flow_error_set(error, ENOMEM,
11324                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11325                                            NULL,
11326                                            "cannot allocate resource memory");
11327                         goto error;
11328                 }
11329                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
11330                 sample_act = &ctx_resource->sample_act[idx];
11331                 action_flags = sample_act->action_flags;
11332                 switch (action_flags) {
11333                 case MLX5_FLOW_ACTION_QUEUE:
11334                         dest_attr[idx]->dest = sample_act->dr_queue_action;
11335                         break;
11336                 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP):
11337                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
11338                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
11339                         dest_attr[idx]->dest_reformat->reformat =
11340                                         sample_act->dr_encap_action;
11341                         dest_attr[idx]->dest_reformat->dest =
11342                                         sample_act->dr_port_id_action;
11343                         break;
11344                 case MLX5_FLOW_ACTION_PORT_ID:
11345                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
11346                         break;
11347                 case MLX5_FLOW_ACTION_JUMP:
11348                         dest_attr[idx]->dest = sample_act->dr_jump_action;
11349                         break;
11350                 default:
11351                         rte_flow_error_set(error, EINVAL,
11352                                            RTE_FLOW_ERROR_TYPE_ACTION,
11353                                            NULL,
11354                                            "unsupported actions type");
11355                         goto error;
11356                 }
11357         }
11358         /* create a dest array actioin */
11359         ret = mlx5_os_flow_dr_create_flow_action_dest_array
11360                                                 (domain,
11361                                                  resource->num_of_dest,
11362                                                  dest_attr,
11363                                                  &resource->action);
11364         if (ret) {
11365                 rte_flow_error_set(error, ENOMEM,
11366                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11367                                    NULL,
11368                                    "cannot create destination array action");
11369                 goto error;
11370         }
11371         resource->idx = res_idx;
11372         resource->dev = dev;
11373         for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11374                 mlx5_free(dest_attr[idx]);
11375         return &resource->entry;
11376 error:
11377         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11378                 flow_dv_sample_sub_actions_release(dev,
11379                                                    &resource->sample_idx[idx]);
11380                 if (dest_attr[idx])
11381                         mlx5_free(dest_attr[idx]);
11382         }
11383         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
11384         return NULL;
11385 }
11386
11387 struct mlx5_list_entry *
11388 flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
11389                             struct mlx5_list_entry *entry __rte_unused,
11390                             void *cb_ctx)
11391 {
11392         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11393         struct rte_eth_dev *dev = ctx->dev;
11394         struct mlx5_flow_dv_dest_array_resource *resource;
11395         struct mlx5_priv *priv = dev->data->dev_private;
11396         struct mlx5_dev_ctx_shared *sh = priv->sh;
11397         uint32_t res_idx = 0;
11398         struct rte_flow_error *error = ctx->error;
11399
11400         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11401                                       &res_idx);
11402         if (!resource) {
11403                 rte_flow_error_set(error, ENOMEM,
11404                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11405                                           NULL,
11406                                           "cannot allocate dest-array memory");
11407                 return NULL;
11408         }
11409         memcpy(resource, entry, sizeof(*resource));
11410         resource->idx = res_idx;
11411         resource->dev = dev;
11412         return &resource->entry;
11413 }
11414
11415 void
11416 flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
11417                                  struct mlx5_list_entry *entry)
11418 {
11419         struct mlx5_flow_dv_dest_array_resource *resource =
11420                         container_of(entry, typeof(*resource), entry);
11421         struct rte_eth_dev *dev = resource->dev;
11422         struct mlx5_priv *priv = dev->data->dev_private;
11423
11424         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
11425 }
11426
11427 /**
11428  * Find existing destination array resource or create and register a new one.
11429  *
11430  * @param[in, out] dev
11431  *   Pointer to rte_eth_dev structure.
11432  * @param[in] ref
11433  *   Pointer to destination array resource reference.
11434  * @parm[in, out] dev_flow
11435  *   Pointer to the dev_flow.
11436  * @param[out] error
11437  *   pointer to error structure.
11438  *
11439  * @return
11440  *   0 on success otherwise -errno and errno is set.
11441  */
11442 static int
11443 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
11444                          struct mlx5_flow_dv_dest_array_resource *ref,
11445                          struct mlx5_flow *dev_flow,
11446                          struct rte_flow_error *error)
11447 {
11448         struct mlx5_flow_dv_dest_array_resource *resource;
11449         struct mlx5_priv *priv = dev->data->dev_private;
11450         struct mlx5_list_entry *entry;
11451         struct mlx5_flow_cb_ctx ctx = {
11452                 .dev = dev,
11453                 .error = error,
11454                 .data = ref,
11455         };
11456
11457         entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
11458         if (!entry)
11459                 return -rte_errno;
11460         resource = container_of(entry, typeof(*resource), entry);
11461         dev_flow->handle->dvh.rix_dest_array = resource->idx;
11462         dev_flow->dv.dest_array_res = resource;
11463         return 0;
11464 }
11465
11466 /**
11467  * Convert Sample action to DV specification.
11468  *
11469  * @param[in] dev
11470  *   Pointer to rte_eth_dev structure.
11471  * @param[in] action
11472  *   Pointer to sample action structure.
11473  * @param[in, out] dev_flow
11474  *   Pointer to the mlx5_flow.
11475  * @param[in] attr
11476  *   Pointer to the flow attributes.
11477  * @param[in, out] num_of_dest
11478  *   Pointer to the num of destination.
11479  * @param[in, out] sample_actions
11480  *   Pointer to sample actions list.
11481  * @param[in, out] res
11482  *   Pointer to sample resource.
11483  * @param[out] error
11484  *   Pointer to the error structure.
11485  *
11486  * @return
11487  *   0 on success, a negative errno value otherwise and rte_errno is set.
11488  */
11489 static int
11490 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
11491                                 const struct rte_flow_action_sample *action,
11492                                 struct mlx5_flow *dev_flow,
11493                                 const struct rte_flow_attr *attr,
11494                                 uint32_t *num_of_dest,
11495                                 void **sample_actions,
11496                                 struct mlx5_flow_dv_sample_resource *res,
11497                                 struct rte_flow_error *error)
11498 {
11499         struct mlx5_priv *priv = dev->data->dev_private;
11500         const struct rte_flow_action *sub_actions;
11501         struct mlx5_flow_sub_actions_list *sample_act;
11502         struct mlx5_flow_sub_actions_idx *sample_idx;
11503         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11504         struct rte_flow *flow = dev_flow->flow;
11505         struct mlx5_flow_rss_desc *rss_desc;
11506         uint64_t action_flags = 0;
11507
11508         MLX5_ASSERT(wks);
11509         rss_desc = &wks->rss_desc;
11510         sample_act = &res->sample_act;
11511         sample_idx = &res->sample_idx;
11512         res->ratio = action->ratio;
11513         sub_actions = action->actions;
11514         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
11515                 int type = sub_actions->type;
11516                 uint32_t pre_rix = 0;
11517                 void *pre_r;
11518                 switch (type) {
11519                 case RTE_FLOW_ACTION_TYPE_QUEUE:
11520                 {
11521                         const struct rte_flow_action_queue *queue;
11522                         struct mlx5_hrxq *hrxq;
11523                         uint32_t hrxq_idx;
11524
11525                         queue = sub_actions->conf;
11526                         rss_desc->queue_num = 1;
11527                         rss_desc->queue[0] = queue->index;
11528                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11529                                                     rss_desc, &hrxq_idx);
11530                         if (!hrxq)
11531                                 return rte_flow_error_set
11532                                         (error, rte_errno,
11533                                          RTE_FLOW_ERROR_TYPE_ACTION,
11534                                          NULL,
11535                                          "cannot create fate queue");
11536                         sample_act->dr_queue_action = hrxq->action;
11537                         sample_idx->rix_hrxq = hrxq_idx;
11538                         sample_actions[sample_act->actions_num++] =
11539                                                 hrxq->action;
11540                         (*num_of_dest)++;
11541                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
11542                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11543                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11544                         dev_flow->handle->fate_action =
11545                                         MLX5_FLOW_FATE_QUEUE;
11546                         break;
11547                 }
11548                 case RTE_FLOW_ACTION_TYPE_RSS:
11549                 {
11550                         struct mlx5_hrxq *hrxq;
11551                         uint32_t hrxq_idx;
11552                         const struct rte_flow_action_rss *rss;
11553                         const uint8_t *rss_key;
11554
11555                         rss = sub_actions->conf;
11556                         memcpy(rss_desc->queue, rss->queue,
11557                                rss->queue_num * sizeof(uint16_t));
11558                         rss_desc->queue_num = rss->queue_num;
11559                         /* NULL RSS key indicates default RSS key. */
11560                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11561                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11562                         /*
11563                          * rss->level and rss.types should be set in advance
11564                          * when expanding items for RSS.
11565                          */
11566                         flow_dv_hashfields_set(dev_flow, rss_desc);
11567                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11568                                                     rss_desc, &hrxq_idx);
11569                         if (!hrxq)
11570                                 return rte_flow_error_set
11571                                         (error, rte_errno,
11572                                          RTE_FLOW_ERROR_TYPE_ACTION,
11573                                          NULL,
11574                                          "cannot create fate queue");
11575                         sample_act->dr_queue_action = hrxq->action;
11576                         sample_idx->rix_hrxq = hrxq_idx;
11577                         sample_actions[sample_act->actions_num++] =
11578                                                 hrxq->action;
11579                         (*num_of_dest)++;
11580                         action_flags |= MLX5_FLOW_ACTION_RSS;
11581                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11582                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11583                         dev_flow->handle->fate_action =
11584                                         MLX5_FLOW_FATE_QUEUE;
11585                         break;
11586                 }
11587                 case RTE_FLOW_ACTION_TYPE_MARK:
11588                 {
11589                         uint32_t tag_be = mlx5_flow_mark_set
11590                                 (((const struct rte_flow_action_mark *)
11591                                 (sub_actions->conf))->id);
11592
11593                         dev_flow->handle->mark = 1;
11594                         pre_rix = dev_flow->handle->dvh.rix_tag;
11595                         /* Save the mark resource before sample */
11596                         pre_r = dev_flow->dv.tag_resource;
11597                         if (flow_dv_tag_resource_register(dev, tag_be,
11598                                                   dev_flow, error))
11599                                 return -rte_errno;
11600                         MLX5_ASSERT(dev_flow->dv.tag_resource);
11601                         sample_act->dr_tag_action =
11602                                 dev_flow->dv.tag_resource->action;
11603                         sample_idx->rix_tag =
11604                                 dev_flow->handle->dvh.rix_tag;
11605                         sample_actions[sample_act->actions_num++] =
11606                                                 sample_act->dr_tag_action;
11607                         /* Recover the mark resource after sample */
11608                         dev_flow->dv.tag_resource = pre_r;
11609                         dev_flow->handle->dvh.rix_tag = pre_rix;
11610                         action_flags |= MLX5_FLOW_ACTION_MARK;
11611                         break;
11612                 }
11613                 case RTE_FLOW_ACTION_TYPE_COUNT:
11614                 {
11615                         if (!flow->counter) {
11616                                 flow->counter =
11617                                         flow_dv_translate_create_counter(dev,
11618                                                 dev_flow, sub_actions->conf,
11619                                                 0);
11620                                 if (!flow->counter)
11621                                         return rte_flow_error_set
11622                                                 (error, rte_errno,
11623                                                 RTE_FLOW_ERROR_TYPE_ACTION,
11624                                                 NULL,
11625                                                 "cannot create counter"
11626                                                 " object.");
11627                         }
11628                         sample_act->dr_cnt_action =
11629                                   (flow_dv_counter_get_by_idx(dev,
11630                                   flow->counter, NULL))->action;
11631                         sample_actions[sample_act->actions_num++] =
11632                                                 sample_act->dr_cnt_action;
11633                         action_flags |= MLX5_FLOW_ACTION_COUNT;
11634                         break;
11635                 }
11636                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
11637                 {
11638                         struct mlx5_flow_dv_port_id_action_resource
11639                                         port_id_resource;
11640                         uint32_t port_id = 0;
11641
11642                         memset(&port_id_resource, 0, sizeof(port_id_resource));
11643                         /* Save the port id resource before sample */
11644                         pre_rix = dev_flow->handle->rix_port_id_action;
11645                         pre_r = dev_flow->dv.port_id_action;
11646                         if (flow_dv_translate_action_port_id(dev, sub_actions,
11647                                                              &port_id, error))
11648                                 return -rte_errno;
11649                         port_id_resource.port_id = port_id;
11650                         if (flow_dv_port_id_action_resource_register
11651                             (dev, &port_id_resource, dev_flow, error))
11652                                 return -rte_errno;
11653                         sample_act->dr_port_id_action =
11654                                 dev_flow->dv.port_id_action->action;
11655                         sample_idx->rix_port_id_action =
11656                                 dev_flow->handle->rix_port_id_action;
11657                         sample_actions[sample_act->actions_num++] =
11658                                                 sample_act->dr_port_id_action;
11659                         /* Recover the port id resource after sample */
11660                         dev_flow->dv.port_id_action = pre_r;
11661                         dev_flow->handle->rix_port_id_action = pre_rix;
11662                         (*num_of_dest)++;
11663                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
11664                         break;
11665                 }
11666                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
11667                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
11668                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
11669                         /* Save the encap resource before sample */
11670                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
11671                         pre_r = dev_flow->dv.encap_decap;
11672                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
11673                                                            dev_flow,
11674                                                            attr->transfer,
11675                                                            error))
11676                                 return -rte_errno;
11677                         sample_act->dr_encap_action =
11678                                 dev_flow->dv.encap_decap->action;
11679                         sample_idx->rix_encap_decap =
11680                                 dev_flow->handle->dvh.rix_encap_decap;
11681                         sample_actions[sample_act->actions_num++] =
11682                                                 sample_act->dr_encap_action;
11683                         /* Recover the encap resource after sample */
11684                         dev_flow->dv.encap_decap = pre_r;
11685                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
11686                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
11687                         break;
11688                 default:
11689                         return rte_flow_error_set(error, EINVAL,
11690                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11691                                 NULL,
11692                                 "Not support for sampler action");
11693                 }
11694         }
11695         sample_act->action_flags = action_flags;
11696         res->ft_id = dev_flow->dv.group;
11697         if (attr->transfer) {
11698                 union {
11699                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
11700                         uint64_t set_action;
11701                 } action_ctx = { .set_action = 0 };
11702
11703                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
11704                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
11705                          MLX5_MODIFICATION_TYPE_SET);
11706                 MLX5_SET(set_action_in, action_ctx.action_in, field,
11707                          MLX5_MODI_META_REG_C_0);
11708                 MLX5_SET(set_action_in, action_ctx.action_in, data,
11709                          priv->vport_meta_tag);
11710                 res->set_action = action_ctx.set_action;
11711         } else if (attr->ingress) {
11712                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
11713         } else {
11714                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
11715         }
11716         return 0;
11717 }
11718
11719 /**
11720  * Convert Sample action to DV specification.
11721  *
11722  * @param[in] dev
11723  *   Pointer to rte_eth_dev structure.
11724  * @param[in, out] dev_flow
11725  *   Pointer to the mlx5_flow.
11726  * @param[in] num_of_dest
11727  *   The num of destination.
11728  * @param[in, out] res
11729  *   Pointer to sample resource.
11730  * @param[in, out] mdest_res
11731  *   Pointer to destination array resource.
11732  * @param[in] sample_actions
11733  *   Pointer to sample path actions list.
11734  * @param[in] action_flags
11735  *   Holds the actions detected until now.
11736  * @param[out] error
11737  *   Pointer to the error structure.
11738  *
11739  * @return
11740  *   0 on success, a negative errno value otherwise and rte_errno is set.
11741  */
11742 static int
11743 flow_dv_create_action_sample(struct rte_eth_dev *dev,
11744                              struct mlx5_flow *dev_flow,
11745                              uint32_t num_of_dest,
11746                              struct mlx5_flow_dv_sample_resource *res,
11747                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
11748                              void **sample_actions,
11749                              uint64_t action_flags,
11750                              struct rte_flow_error *error)
11751 {
11752         /* update normal path action resource into last index of array */
11753         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
11754         struct mlx5_flow_sub_actions_list *sample_act =
11755                                         &mdest_res->sample_act[dest_index];
11756         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11757         struct mlx5_flow_rss_desc *rss_desc;
11758         uint32_t normal_idx = 0;
11759         struct mlx5_hrxq *hrxq;
11760         uint32_t hrxq_idx;
11761
11762         MLX5_ASSERT(wks);
11763         rss_desc = &wks->rss_desc;
11764         if (num_of_dest > 1) {
11765                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
11766                         /* Handle QP action for mirroring */
11767                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11768                                                     rss_desc, &hrxq_idx);
11769                         if (!hrxq)
11770                                 return rte_flow_error_set
11771                                      (error, rte_errno,
11772                                       RTE_FLOW_ERROR_TYPE_ACTION,
11773                                       NULL,
11774                                       "cannot create rx queue");
11775                         normal_idx++;
11776                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
11777                         sample_act->dr_queue_action = hrxq->action;
11778                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11779                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11780                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
11781                 }
11782                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
11783                         normal_idx++;
11784                         mdest_res->sample_idx[dest_index].rix_encap_decap =
11785                                 dev_flow->handle->dvh.rix_encap_decap;
11786                         sample_act->dr_encap_action =
11787                                 dev_flow->dv.encap_decap->action;
11788                         dev_flow->handle->dvh.rix_encap_decap = 0;
11789                 }
11790                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
11791                         normal_idx++;
11792                         mdest_res->sample_idx[dest_index].rix_port_id_action =
11793                                 dev_flow->handle->rix_port_id_action;
11794                         sample_act->dr_port_id_action =
11795                                 dev_flow->dv.port_id_action->action;
11796                         dev_flow->handle->rix_port_id_action = 0;
11797                 }
11798                 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) {
11799                         normal_idx++;
11800                         mdest_res->sample_idx[dest_index].rix_jump =
11801                                 dev_flow->handle->rix_jump;
11802                         sample_act->dr_jump_action =
11803                                 dev_flow->dv.jump->action;
11804                         dev_flow->handle->rix_jump = 0;
11805                 }
11806                 sample_act->actions_num = normal_idx;
11807                 /* update sample action resource into first index of array */
11808                 mdest_res->ft_type = res->ft_type;
11809                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
11810                                 sizeof(struct mlx5_flow_sub_actions_idx));
11811                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
11812                                 sizeof(struct mlx5_flow_sub_actions_list));
11813                 mdest_res->num_of_dest = num_of_dest;
11814                 if (flow_dv_dest_array_resource_register(dev, mdest_res,
11815                                                          dev_flow, error))
11816                         return rte_flow_error_set(error, EINVAL,
11817                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11818                                                   NULL, "can't create sample "
11819                                                   "action");
11820         } else {
11821                 res->sub_actions = sample_actions;
11822                 if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
11823                         return rte_flow_error_set(error, EINVAL,
11824                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11825                                                   NULL,
11826                                                   "can't create sample action");
11827         }
11828         return 0;
11829 }
11830
11831 /**
11832  * Remove an ASO age action from age actions list.
11833  *
11834  * @param[in] dev
11835  *   Pointer to the Ethernet device structure.
11836  * @param[in] age
11837  *   Pointer to the aso age action handler.
11838  */
11839 static void
11840 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev,
11841                                 struct mlx5_aso_age_action *age)
11842 {
11843         struct mlx5_age_info *age_info;
11844         struct mlx5_age_param *age_param = &age->age_params;
11845         struct mlx5_priv *priv = dev->data->dev_private;
11846         uint16_t expected = AGE_CANDIDATE;
11847
11848         age_info = GET_PORT_AGE_INFO(priv);
11849         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
11850                                          AGE_FREE, false, __ATOMIC_RELAXED,
11851                                          __ATOMIC_RELAXED)) {
11852                 /**
11853                  * We need the lock even it is age timeout,
11854                  * since age action may still in process.
11855                  */
11856                 rte_spinlock_lock(&age_info->aged_sl);
11857                 LIST_REMOVE(age, next);
11858                 rte_spinlock_unlock(&age_info->aged_sl);
11859                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
11860         }
11861 }
11862
11863 /**
11864  * Release an ASO age action.
11865  *
11866  * @param[in] dev
11867  *   Pointer to the Ethernet device structure.
11868  * @param[in] age_idx
11869  *   Index of ASO age action to release.
11870  * @param[in] flow
11871  *   True if the release operation is during flow destroy operation.
11872  *   False if the release operation is during action destroy operation.
11873  *
11874  * @return
11875  *   0 when age action was removed, otherwise the number of references.
11876  */
11877 static int
11878 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
11879 {
11880         struct mlx5_priv *priv = dev->data->dev_private;
11881         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11882         struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx);
11883         uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED);
11884
11885         if (!ret) {
11886                 flow_dv_aso_age_remove_from_age(dev, age);
11887                 rte_spinlock_lock(&mng->free_sl);
11888                 LIST_INSERT_HEAD(&mng->free, age, next);
11889                 rte_spinlock_unlock(&mng->free_sl);
11890         }
11891         return ret;
11892 }
11893
11894 /**
11895  * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
11896  *
11897  * @param[in] dev
11898  *   Pointer to the Ethernet device structure.
11899  *
11900  * @return
11901  *   0 on success, otherwise negative errno value and rte_errno is set.
11902  */
11903 static int
11904 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
11905 {
11906         struct mlx5_priv *priv = dev->data->dev_private;
11907         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11908         void *old_pools = mng->pools;
11909         uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
11910         uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
11911         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
11912
11913         if (!pools) {
11914                 rte_errno = ENOMEM;
11915                 return -ENOMEM;
11916         }
11917         if (old_pools) {
11918                 memcpy(pools, old_pools,
11919                        mng->n * sizeof(struct mlx5_flow_counter_pool *));
11920                 mlx5_free(old_pools);
11921         } else {
11922                 /* First ASO flow hit allocation - starting ASO data-path. */
11923                 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh);
11924
11925                 if (ret) {
11926                         mlx5_free(pools);
11927                         return ret;
11928                 }
11929         }
11930         mng->n = resize;
11931         mng->pools = pools;
11932         return 0;
11933 }
11934
11935 /**
11936  * Create and initialize a new ASO aging pool.
11937  *
11938  * @param[in] dev
11939  *   Pointer to the Ethernet device structure.
11940  * @param[out] age_free
11941  *   Where to put the pointer of a new age action.
11942  *
11943  * @return
11944  *   The age actions pool pointer and @p age_free is set on success,
11945  *   NULL otherwise and rte_errno is set.
11946  */
11947 static struct mlx5_aso_age_pool *
11948 flow_dv_age_pool_create(struct rte_eth_dev *dev,
11949                         struct mlx5_aso_age_action **age_free)
11950 {
11951         struct mlx5_priv *priv = dev->data->dev_private;
11952         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11953         struct mlx5_aso_age_pool *pool = NULL;
11954         struct mlx5_devx_obj *obj = NULL;
11955         uint32_t i;
11956
11957         obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->ctx,
11958                                                     priv->sh->pdn);
11959         if (!obj) {
11960                 rte_errno = ENODATA;
11961                 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX.");
11962                 return NULL;
11963         }
11964         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
11965         if (!pool) {
11966                 claim_zero(mlx5_devx_cmd_destroy(obj));
11967                 rte_errno = ENOMEM;
11968                 return NULL;
11969         }
11970         pool->flow_hit_aso_obj = obj;
11971         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
11972         rte_spinlock_lock(&mng->resize_sl);
11973         pool->index = mng->next;
11974         /* Resize pools array if there is no room for the new pool in it. */
11975         if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) {
11976                 claim_zero(mlx5_devx_cmd_destroy(obj));
11977                 mlx5_free(pool);
11978                 rte_spinlock_unlock(&mng->resize_sl);
11979                 return NULL;
11980         }
11981         mng->pools[pool->index] = pool;
11982         mng->next++;
11983         rte_spinlock_unlock(&mng->resize_sl);
11984         /* Assign the first action in the new pool, the rest go to free list. */
11985         *age_free = &pool->actions[0];
11986         for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) {
11987                 pool->actions[i].offset = i;
11988                 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next);
11989         }
11990         return pool;
11991 }
11992
11993 /**
11994  * Allocate a ASO aging bit.
11995  *
11996  * @param[in] dev
11997  *   Pointer to the Ethernet device structure.
11998  * @param[out] error
11999  *   Pointer to the error structure.
12000  *
12001  * @return
12002  *   Index to ASO age action on success, 0 otherwise and rte_errno is set.
12003  */
12004 static uint32_t
12005 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12006 {
12007         struct mlx5_priv *priv = dev->data->dev_private;
12008         const struct mlx5_aso_age_pool *pool;
12009         struct mlx5_aso_age_action *age_free = NULL;
12010         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12011
12012         MLX5_ASSERT(mng);
12013         /* Try to get the next free age action bit. */
12014         rte_spinlock_lock(&mng->free_sl);
12015         age_free = LIST_FIRST(&mng->free);
12016         if (age_free) {
12017                 LIST_REMOVE(age_free, next);
12018         } else if (!flow_dv_age_pool_create(dev, &age_free)) {
12019                 rte_spinlock_unlock(&mng->free_sl);
12020                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12021                                    NULL, "failed to create ASO age pool");
12022                 return 0; /* 0 is an error. */
12023         }
12024         rte_spinlock_unlock(&mng->free_sl);
12025         pool = container_of
12026           ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL])
12027                   (age_free - age_free->offset), const struct mlx5_aso_age_pool,
12028                                                                        actions);
12029         if (!age_free->dr_action) {
12030                 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0,
12031                                                  error);
12032
12033                 if (reg_c < 0) {
12034                         rte_flow_error_set(error, rte_errno,
12035                                            RTE_FLOW_ERROR_TYPE_ACTION,
12036                                            NULL, "failed to get reg_c "
12037                                            "for ASO flow hit");
12038                         return 0; /* 0 is an error. */
12039                 }
12040 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
12041                 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso
12042                                 (priv->sh->rx_domain,
12043                                  pool->flow_hit_aso_obj->obj, age_free->offset,
12044                                  MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET,
12045                                  (reg_c - REG_C_0));
12046 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
12047                 if (!age_free->dr_action) {
12048                         rte_errno = errno;
12049                         rte_spinlock_lock(&mng->free_sl);
12050                         LIST_INSERT_HEAD(&mng->free, age_free, next);
12051                         rte_spinlock_unlock(&mng->free_sl);
12052                         rte_flow_error_set(error, rte_errno,
12053                                            RTE_FLOW_ERROR_TYPE_ACTION,
12054                                            NULL, "failed to create ASO "
12055                                            "flow hit action");
12056                         return 0; /* 0 is an error. */
12057                 }
12058         }
12059         __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED);
12060         return pool->index | ((age_free->offset + 1) << 16);
12061 }
12062
12063 /**
12064  * Initialize flow ASO age parameters.
12065  *
12066  * @param[in] dev
12067  *   Pointer to rte_eth_dev structure.
12068  * @param[in] age_idx
12069  *   Index of ASO age action.
12070  * @param[in] context
12071  *   Pointer to flow counter age context.
12072  * @param[in] timeout
12073  *   Aging timeout in seconds.
12074  *
12075  */
12076 static void
12077 flow_dv_aso_age_params_init(struct rte_eth_dev *dev,
12078                             uint32_t age_idx,
12079                             void *context,
12080                             uint32_t timeout)
12081 {
12082         struct mlx5_aso_age_action *aso_age;
12083
12084         aso_age = flow_aso_age_get_by_idx(dev, age_idx);
12085         MLX5_ASSERT(aso_age);
12086         aso_age->age_params.context = context;
12087         aso_age->age_params.timeout = timeout;
12088         aso_age->age_params.port_id = dev->data->port_id;
12089         __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0,
12090                          __ATOMIC_RELAXED);
12091         __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE,
12092                          __ATOMIC_RELAXED);
12093 }
12094
12095 static void
12096 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask,
12097                                const struct rte_flow_item_integrity *value,
12098                                void *headers_m, void *headers_v)
12099 {
12100         if (mask->l4_ok) {
12101                 /* application l4_ok filter aggregates all hardware l4 filters
12102                  * therefore hw l4_checksum_ok must be implicitly added here.
12103                  */
12104                 struct rte_flow_item_integrity local_item;
12105
12106                 local_item.l4_csum_ok = 1;
12107                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
12108                          local_item.l4_csum_ok);
12109                 if (value->l4_ok) {
12110                         /* application l4_ok = 1 matches sets both hw flags
12111                          * l4_ok and l4_checksum_ok flags to 1.
12112                          */
12113                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12114                                  l4_checksum_ok, local_item.l4_csum_ok);
12115                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok,
12116                                  mask->l4_ok);
12117                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok,
12118                                  value->l4_ok);
12119                 } else {
12120                         /* application l4_ok = 0 matches on hw flag
12121                          * l4_checksum_ok = 0 only.
12122                          */
12123                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12124                                  l4_checksum_ok, 0);
12125                 }
12126         } else if (mask->l4_csum_ok) {
12127                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
12128                          mask->l4_csum_ok);
12129                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
12130                          value->l4_csum_ok);
12131         }
12132 }
12133
12134 static void
12135 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask,
12136                                const struct rte_flow_item_integrity *value,
12137                                void *headers_m, void *headers_v,
12138                                bool is_ipv4)
12139 {
12140         if (mask->l3_ok) {
12141                 /* application l3_ok filter aggregates all hardware l3 filters
12142                  * therefore hw ipv4_checksum_ok must be implicitly added here.
12143                  */
12144                 struct rte_flow_item_integrity local_item;
12145
12146                 local_item.ipv4_csum_ok = !!is_ipv4;
12147                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
12148                          local_item.ipv4_csum_ok);
12149                 if (value->l3_ok) {
12150                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12151                                  ipv4_checksum_ok, local_item.ipv4_csum_ok);
12152                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok,
12153                                  mask->l3_ok);
12154                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok,
12155                                  value->l3_ok);
12156                 } else {
12157                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12158                                  ipv4_checksum_ok, 0);
12159                 }
12160         } else if (mask->ipv4_csum_ok) {
12161                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
12162                          mask->ipv4_csum_ok);
12163                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
12164                          value->ipv4_csum_ok);
12165         }
12166 }
12167
12168 static void
12169 flow_dv_translate_item_integrity(void *matcher, void *key,
12170                                  const struct rte_flow_item *head_item,
12171                                  const struct rte_flow_item *integrity_item)
12172 {
12173         const struct rte_flow_item_integrity *mask = integrity_item->mask;
12174         const struct rte_flow_item_integrity *value = integrity_item->spec;
12175         const struct rte_flow_item *tunnel_item, *end_item, *item;
12176         void *headers_m;
12177         void *headers_v;
12178         uint32_t l3_protocol;
12179
12180         if (!value)
12181                 return;
12182         if (!mask)
12183                 mask = &rte_flow_item_integrity_mask;
12184         if (value->level > 1) {
12185                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12186                                          inner_headers);
12187                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
12188         } else {
12189                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12190                                          outer_headers);
12191                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
12192         }
12193         tunnel_item = mlx5_flow_find_tunnel_item(head_item);
12194         if (value->level > 1) {
12195                 /* tunnel item was verified during the item validation */
12196                 item = tunnel_item;
12197                 end_item = mlx5_find_end_item(tunnel_item);
12198         } else {
12199                 item = head_item;
12200                 end_item = tunnel_item ? tunnel_item :
12201                            mlx5_find_end_item(integrity_item);
12202         }
12203         l3_protocol = mask->l3_ok ?
12204                       mlx5_flow_locate_proto_l3(&item, end_item) : 0;
12205         flow_dv_translate_integrity_l3(mask, value, headers_m, headers_v,
12206                                        l3_protocol == RTE_ETHER_TYPE_IPV4);
12207         flow_dv_translate_integrity_l4(mask, value, headers_m, headers_v);
12208 }
12209
12210 /**
12211  * Prepares DV flow counter with aging configuration.
12212  * Gets it by index when exists, creates a new one when doesn't.
12213  *
12214  * @param[in] dev
12215  *   Pointer to rte_eth_dev structure.
12216  * @param[in] dev_flow
12217  *   Pointer to the mlx5_flow.
12218  * @param[in, out] flow
12219  *   Pointer to the sub flow.
12220  * @param[in] count
12221  *   Pointer to the counter action configuration.
12222  * @param[in] age
12223  *   Pointer to the aging action configuration.
12224  * @param[out] error
12225  *   Pointer to the error structure.
12226  *
12227  * @return
12228  *   Pointer to the counter, NULL otherwise.
12229  */
12230 static struct mlx5_flow_counter *
12231 flow_dv_prepare_counter(struct rte_eth_dev *dev,
12232                         struct mlx5_flow *dev_flow,
12233                         struct rte_flow *flow,
12234                         const struct rte_flow_action_count *count,
12235                         const struct rte_flow_action_age *age,
12236                         struct rte_flow_error *error)
12237 {
12238         if (!flow->counter) {
12239                 flow->counter = flow_dv_translate_create_counter(dev, dev_flow,
12240                                                                  count, age);
12241                 if (!flow->counter) {
12242                         rte_flow_error_set(error, rte_errno,
12243                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12244                                            "cannot create counter object.");
12245                         return NULL;
12246                 }
12247         }
12248         return flow_dv_counter_get_by_idx(dev, flow->counter, NULL);
12249 }
12250
12251 /*
12252  * Release an ASO CT action by its own device.
12253  *
12254  * @param[in] dev
12255  *   Pointer to the Ethernet device structure.
12256  * @param[in] idx
12257  *   Index of ASO CT action to release.
12258  *
12259  * @return
12260  *   0 when CT action was removed, otherwise the number of references.
12261  */
12262 static inline int
12263 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx)
12264 {
12265         struct mlx5_priv *priv = dev->data->dev_private;
12266         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12267         uint32_t ret;
12268         struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12269         enum mlx5_aso_ct_state state =
12270                         __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
12271
12272         /* Cannot release when CT is in the ASO SQ. */
12273         if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY)
12274                 return -1;
12275         ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED);
12276         if (!ret) {
12277                 if (ct->dr_action_orig) {
12278 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12279                         claim_zero(mlx5_glue->destroy_flow_action
12280                                         (ct->dr_action_orig));
12281 #endif
12282                         ct->dr_action_orig = NULL;
12283                 }
12284                 if (ct->dr_action_rply) {
12285 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12286                         claim_zero(mlx5_glue->destroy_flow_action
12287                                         (ct->dr_action_rply));
12288 #endif
12289                         ct->dr_action_rply = NULL;
12290                 }
12291                 /* Clear the state to free, no need in 1st allocation. */
12292                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE);
12293                 rte_spinlock_lock(&mng->ct_sl);
12294                 LIST_INSERT_HEAD(&mng->free_cts, ct, next);
12295                 rte_spinlock_unlock(&mng->ct_sl);
12296         }
12297         return (int)ret;
12298 }
12299
12300 static inline int
12301 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx,
12302                        struct rte_flow_error *error)
12303 {
12304         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx);
12305         uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx);
12306         struct rte_eth_dev *owndev = &rte_eth_devices[owner];
12307         int ret;
12308
12309         MLX5_ASSERT(owner < RTE_MAX_ETHPORTS);
12310         if (dev->data->dev_started != 1)
12311                 return rte_flow_error_set(error, EAGAIN,
12312                                           RTE_FLOW_ERROR_TYPE_ACTION,
12313                                           NULL,
12314                                           "Indirect CT action cannot be destroyed when the port is stopped");
12315         ret = flow_dv_aso_ct_dev_release(owndev, idx);
12316         if (ret < 0)
12317                 return rte_flow_error_set(error, EAGAIN,
12318                                           RTE_FLOW_ERROR_TYPE_ACTION,
12319                                           NULL,
12320                                           "Current state prevents indirect CT action from being destroyed");
12321         return ret;
12322 }
12323
12324 /*
12325  * Resize the ASO CT pools array by 64 pools.
12326  *
12327  * @param[in] dev
12328  *   Pointer to the Ethernet device structure.
12329  *
12330  * @return
12331  *   0 on success, otherwise negative errno value and rte_errno is set.
12332  */
12333 static int
12334 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev)
12335 {
12336         struct mlx5_priv *priv = dev->data->dev_private;
12337         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12338         void *old_pools = mng->pools;
12339         /* Magic number now, need a macro. */
12340         uint32_t resize = mng->n + 64;
12341         uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize;
12342         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
12343
12344         if (!pools) {
12345                 rte_errno = ENOMEM;
12346                 return -rte_errno;
12347         }
12348         rte_rwlock_write_lock(&mng->resize_rwl);
12349         /* ASO SQ/QP was already initialized in the startup. */
12350         if (old_pools) {
12351                 /* Realloc could be an alternative choice. */
12352                 rte_memcpy(pools, old_pools,
12353                            mng->n * sizeof(struct mlx5_aso_ct_pool *));
12354                 mlx5_free(old_pools);
12355         }
12356         mng->n = resize;
12357         mng->pools = pools;
12358         rte_rwlock_write_unlock(&mng->resize_rwl);
12359         return 0;
12360 }
12361
12362 /*
12363  * Create and initialize a new ASO CT pool.
12364  *
12365  * @param[in] dev
12366  *   Pointer to the Ethernet device structure.
12367  * @param[out] ct_free
12368  *   Where to put the pointer of a new CT action.
12369  *
12370  * @return
12371  *   The CT actions pool pointer and @p ct_free is set on success,
12372  *   NULL otherwise and rte_errno is set.
12373  */
12374 static struct mlx5_aso_ct_pool *
12375 flow_dv_ct_pool_create(struct rte_eth_dev *dev,
12376                        struct mlx5_aso_ct_action **ct_free)
12377 {
12378         struct mlx5_priv *priv = dev->data->dev_private;
12379         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12380         struct mlx5_aso_ct_pool *pool = NULL;
12381         struct mlx5_devx_obj *obj = NULL;
12382         uint32_t i;
12383         uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL);
12384
12385         obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->ctx,
12386                                                 priv->sh->pdn, log_obj_size);
12387         if (!obj) {
12388                 rte_errno = ENODATA;
12389                 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
12390                 return NULL;
12391         }
12392         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
12393         if (!pool) {
12394                 rte_errno = ENOMEM;
12395                 claim_zero(mlx5_devx_cmd_destroy(obj));
12396                 return NULL;
12397         }
12398         pool->devx_obj = obj;
12399         pool->index = mng->next;
12400         /* Resize pools array if there is no room for the new pool in it. */
12401         if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) {
12402                 claim_zero(mlx5_devx_cmd_destroy(obj));
12403                 mlx5_free(pool);
12404                 return NULL;
12405         }
12406         mng->pools[pool->index] = pool;
12407         mng->next++;
12408         /* Assign the first action in the new pool, the rest go to free list. */
12409         *ct_free = &pool->actions[0];
12410         /* Lock outside, the list operation is safe here. */
12411         for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) {
12412                 /* refcnt is 0 when allocating the memory. */
12413                 pool->actions[i].offset = i;
12414                 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next);
12415         }
12416         return pool;
12417 }
12418
12419 /*
12420  * Allocate a ASO CT action from free list.
12421  *
12422  * @param[in] dev
12423  *   Pointer to the Ethernet device structure.
12424  * @param[out] error
12425  *   Pointer to the error structure.
12426  *
12427  * @return
12428  *   Index to ASO CT action on success, 0 otherwise and rte_errno is set.
12429  */
12430 static uint32_t
12431 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12432 {
12433         struct mlx5_priv *priv = dev->data->dev_private;
12434         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12435         struct mlx5_aso_ct_action *ct = NULL;
12436         struct mlx5_aso_ct_pool *pool;
12437         uint8_t reg_c;
12438         uint32_t ct_idx;
12439
12440         MLX5_ASSERT(mng);
12441         if (!priv->config.devx) {
12442                 rte_errno = ENOTSUP;
12443                 return 0;
12444         }
12445         /* Get a free CT action, if no, a new pool will be created. */
12446         rte_spinlock_lock(&mng->ct_sl);
12447         ct = LIST_FIRST(&mng->free_cts);
12448         if (ct) {
12449                 LIST_REMOVE(ct, next);
12450         } else if (!flow_dv_ct_pool_create(dev, &ct)) {
12451                 rte_spinlock_unlock(&mng->ct_sl);
12452                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12453                                    NULL, "failed to create ASO CT pool");
12454                 return 0;
12455         }
12456         rte_spinlock_unlock(&mng->ct_sl);
12457         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
12458         ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset);
12459         /* 0: inactive, 1: created, 2+: used by flows. */
12460         __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED);
12461         reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error);
12462         if (!ct->dr_action_orig) {
12463 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12464                 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso
12465                         (priv->sh->rx_domain, pool->devx_obj->obj,
12466                          ct->offset,
12467                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR,
12468                          reg_c - REG_C_0);
12469 #else
12470                 RTE_SET_USED(reg_c);
12471 #endif
12472                 if (!ct->dr_action_orig) {
12473                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12474                         rte_flow_error_set(error, rte_errno,
12475                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12476                                            "failed to create ASO CT action");
12477                         return 0;
12478                 }
12479         }
12480         if (!ct->dr_action_rply) {
12481 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12482                 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso
12483                         (priv->sh->rx_domain, pool->devx_obj->obj,
12484                          ct->offset,
12485                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER,
12486                          reg_c - REG_C_0);
12487 #endif
12488                 if (!ct->dr_action_rply) {
12489                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12490                         rte_flow_error_set(error, rte_errno,
12491                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12492                                            "failed to create ASO CT action");
12493                         return 0;
12494                 }
12495         }
12496         return ct_idx;
12497 }
12498
12499 /*
12500  * Create a conntrack object with context and actions by using ASO mechanism.
12501  *
12502  * @param[in] dev
12503  *   Pointer to rte_eth_dev structure.
12504  * @param[in] pro
12505  *   Pointer to conntrack information profile.
12506  * @param[out] error
12507  *   Pointer to the error structure.
12508  *
12509  * @return
12510  *   Index to conntrack object on success, 0 otherwise.
12511  */
12512 static uint32_t
12513 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev,
12514                                    const struct rte_flow_action_conntrack *pro,
12515                                    struct rte_flow_error *error)
12516 {
12517         struct mlx5_priv *priv = dev->data->dev_private;
12518         struct mlx5_dev_ctx_shared *sh = priv->sh;
12519         struct mlx5_aso_ct_action *ct;
12520         uint32_t idx;
12521
12522         if (!sh->ct_aso_en)
12523                 return rte_flow_error_set(error, ENOTSUP,
12524                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12525                                           "Connection is not supported");
12526         idx = flow_dv_aso_ct_alloc(dev, error);
12527         if (!idx)
12528                 return rte_flow_error_set(error, rte_errno,
12529                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12530                                           "Failed to allocate CT object");
12531         ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12532         if (mlx5_aso_ct_update_by_wqe(sh, ct, pro))
12533                 return rte_flow_error_set(error, EBUSY,
12534                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12535                                           "Failed to update CT");
12536         ct->is_original = !!pro->is_original_dir;
12537         ct->peer = pro->peer_port;
12538         return idx;
12539 }
12540
12541 /**
12542  * Fill the flow with DV spec, lock free
12543  * (mutex should be acquired by caller).
12544  *
12545  * @param[in] dev
12546  *   Pointer to rte_eth_dev structure.
12547  * @param[in, out] dev_flow
12548  *   Pointer to the sub flow.
12549  * @param[in] attr
12550  *   Pointer to the flow attributes.
12551  * @param[in] items
12552  *   Pointer to the list of items.
12553  * @param[in] actions
12554  *   Pointer to the list of actions.
12555  * @param[out] error
12556  *   Pointer to the error structure.
12557  *
12558  * @return
12559  *   0 on success, a negative errno value otherwise and rte_errno is set.
12560  */
12561 static int
12562 flow_dv_translate(struct rte_eth_dev *dev,
12563                   struct mlx5_flow *dev_flow,
12564                   const struct rte_flow_attr *attr,
12565                   const struct rte_flow_item items[],
12566                   const struct rte_flow_action actions[],
12567                   struct rte_flow_error *error)
12568 {
12569         struct mlx5_priv *priv = dev->data->dev_private;
12570         struct mlx5_dev_config *dev_conf = &priv->config;
12571         struct rte_flow *flow = dev_flow->flow;
12572         struct mlx5_flow_handle *handle = dev_flow->handle;
12573         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12574         struct mlx5_flow_rss_desc *rss_desc;
12575         uint64_t item_flags = 0;
12576         uint64_t last_item = 0;
12577         uint64_t action_flags = 0;
12578         struct mlx5_flow_dv_matcher matcher = {
12579                 .mask = {
12580                         .size = sizeof(matcher.mask.buf),
12581                 },
12582         };
12583         int actions_n = 0;
12584         bool actions_end = false;
12585         union {
12586                 struct mlx5_flow_dv_modify_hdr_resource res;
12587                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
12588                             sizeof(struct mlx5_modification_cmd) *
12589                             (MLX5_MAX_MODIFY_NUM + 1)];
12590         } mhdr_dummy;
12591         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
12592         const struct rte_flow_action_count *count = NULL;
12593         const struct rte_flow_action_age *non_shared_age = NULL;
12594         union flow_dv_attr flow_attr = { .attr = 0 };
12595         uint32_t tag_be;
12596         union mlx5_flow_tbl_key tbl_key;
12597         uint32_t modify_action_position = UINT32_MAX;
12598         void *match_mask = matcher.mask.buf;
12599         void *match_value = dev_flow->dv.value.buf;
12600         uint8_t next_protocol = 0xff;
12601         struct rte_vlan_hdr vlan = { 0 };
12602         struct mlx5_flow_dv_dest_array_resource mdest_res;
12603         struct mlx5_flow_dv_sample_resource sample_res;
12604         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
12605         const struct rte_flow_action_sample *sample = NULL;
12606         struct mlx5_flow_sub_actions_list *sample_act;
12607         uint32_t sample_act_pos = UINT32_MAX;
12608         uint32_t age_act_pos = UINT32_MAX;
12609         uint32_t num_of_dest = 0;
12610         int tmp_actions_n = 0;
12611         uint32_t table;
12612         int ret = 0;
12613         const struct mlx5_flow_tunnel *tunnel = NULL;
12614         struct flow_grp_info grp_info = {
12615                 .external = !!dev_flow->external,
12616                 .transfer = !!attr->transfer,
12617                 .fdb_def_rule = !!priv->fdb_def_rule,
12618                 .skip_scale = dev_flow->skip_scale &
12619                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
12620                 .std_tbl_fix = true,
12621         };
12622         const struct rte_flow_item *head_item = items;
12623
12624         if (!wks)
12625                 return rte_flow_error_set(error, ENOMEM,
12626                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12627                                           NULL,
12628                                           "failed to push flow workspace");
12629         rss_desc = &wks->rss_desc;
12630         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
12631         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
12632         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12633                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12634         /* update normal path action resource into last index of array */
12635         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
12636         if (is_tunnel_offload_active(dev)) {
12637                 if (dev_flow->tunnel) {
12638                         RTE_VERIFY(dev_flow->tof_type ==
12639                                    MLX5_TUNNEL_OFFLOAD_MISS_RULE);
12640                         tunnel = dev_flow->tunnel;
12641                 } else {
12642                         tunnel = mlx5_get_tof(items, actions,
12643                                               &dev_flow->tof_type);
12644                         dev_flow->tunnel = tunnel;
12645                 }
12646                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
12647                                         (dev, attr, tunnel, dev_flow->tof_type);
12648         }
12649         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12650                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12651         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
12652                                        &grp_info, error);
12653         if (ret)
12654                 return ret;
12655         dev_flow->dv.group = table;
12656         if (attr->transfer)
12657                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12658         /* number of actions must be set to 0 in case of dirty stack. */
12659         mhdr_res->actions_num = 0;
12660         if (is_flow_tunnel_match_rule(dev_flow->tof_type)) {
12661                 /*
12662                  * do not add decap action if match rule drops packet
12663                  * HW rejects rules with decap & drop
12664                  *
12665                  * if tunnel match rule was inserted before matching tunnel set
12666                  * rule flow table used in the match rule must be registered.
12667                  * current implementation handles that in the
12668                  * flow_dv_match_register() at the function end.
12669                  */
12670                 bool add_decap = true;
12671                 const struct rte_flow_action *ptr = actions;
12672
12673                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
12674                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
12675                                 add_decap = false;
12676                                 break;
12677                         }
12678                 }
12679                 if (add_decap) {
12680                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12681                                                            attr->transfer,
12682                                                            error))
12683                                 return -rte_errno;
12684                         dev_flow->dv.actions[actions_n++] =
12685                                         dev_flow->dv.encap_decap->action;
12686                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12687                 }
12688         }
12689         for (; !actions_end ; actions++) {
12690                 const struct rte_flow_action_queue *queue;
12691                 const struct rte_flow_action_rss *rss;
12692                 const struct rte_flow_action *action = actions;
12693                 const uint8_t *rss_key;
12694                 struct mlx5_flow_tbl_resource *tbl;
12695                 struct mlx5_aso_age_action *age_act;
12696                 struct mlx5_flow_counter *cnt_act;
12697                 uint32_t port_id = 0;
12698                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
12699                 int action_type = actions->type;
12700                 const struct rte_flow_action *found_action = NULL;
12701                 uint32_t jump_group = 0;
12702                 uint32_t owner_idx;
12703                 struct mlx5_aso_ct_action *ct;
12704
12705                 if (!mlx5_flow_os_action_supported(action_type))
12706                         return rte_flow_error_set(error, ENOTSUP,
12707                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12708                                                   actions,
12709                                                   "action not supported");
12710                 switch (action_type) {
12711                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
12712                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
12713                         break;
12714                 case RTE_FLOW_ACTION_TYPE_VOID:
12715                         break;
12716                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
12717                         if (flow_dv_translate_action_port_id(dev, action,
12718                                                              &port_id, error))
12719                                 return -rte_errno;
12720                         port_id_resource.port_id = port_id;
12721                         MLX5_ASSERT(!handle->rix_port_id_action);
12722                         if (flow_dv_port_id_action_resource_register
12723                             (dev, &port_id_resource, dev_flow, error))
12724                                 return -rte_errno;
12725                         dev_flow->dv.actions[actions_n++] =
12726                                         dev_flow->dv.port_id_action->action;
12727                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12728                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
12729                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12730                         num_of_dest++;
12731                         break;
12732                 case RTE_FLOW_ACTION_TYPE_FLAG:
12733                         action_flags |= MLX5_FLOW_ACTION_FLAG;
12734                         dev_flow->handle->mark = 1;
12735                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12736                                 struct rte_flow_action_mark mark = {
12737                                         .id = MLX5_FLOW_MARK_DEFAULT,
12738                                 };
12739
12740                                 if (flow_dv_convert_action_mark(dev, &mark,
12741                                                                 mhdr_res,
12742                                                                 error))
12743                                         return -rte_errno;
12744                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12745                                 break;
12746                         }
12747                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
12748                         /*
12749                          * Only one FLAG or MARK is supported per device flow
12750                          * right now. So the pointer to the tag resource must be
12751                          * zero before the register process.
12752                          */
12753                         MLX5_ASSERT(!handle->dvh.rix_tag);
12754                         if (flow_dv_tag_resource_register(dev, tag_be,
12755                                                           dev_flow, error))
12756                                 return -rte_errno;
12757                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12758                         dev_flow->dv.actions[actions_n++] =
12759                                         dev_flow->dv.tag_resource->action;
12760                         break;
12761                 case RTE_FLOW_ACTION_TYPE_MARK:
12762                         action_flags |= MLX5_FLOW_ACTION_MARK;
12763                         dev_flow->handle->mark = 1;
12764                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12765                                 const struct rte_flow_action_mark *mark =
12766                                         (const struct rte_flow_action_mark *)
12767                                                 actions->conf;
12768
12769                                 if (flow_dv_convert_action_mark(dev, mark,
12770                                                                 mhdr_res,
12771                                                                 error))
12772                                         return -rte_errno;
12773                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12774                                 break;
12775                         }
12776                         /* Fall-through */
12777                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
12778                         /* Legacy (non-extensive) MARK action. */
12779                         tag_be = mlx5_flow_mark_set
12780                               (((const struct rte_flow_action_mark *)
12781                                (actions->conf))->id);
12782                         MLX5_ASSERT(!handle->dvh.rix_tag);
12783                         if (flow_dv_tag_resource_register(dev, tag_be,
12784                                                           dev_flow, error))
12785                                 return -rte_errno;
12786                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12787                         dev_flow->dv.actions[actions_n++] =
12788                                         dev_flow->dv.tag_resource->action;
12789                         break;
12790                 case RTE_FLOW_ACTION_TYPE_SET_META:
12791                         if (flow_dv_convert_action_set_meta
12792                                 (dev, mhdr_res, attr,
12793                                  (const struct rte_flow_action_set_meta *)
12794                                   actions->conf, error))
12795                                 return -rte_errno;
12796                         action_flags |= MLX5_FLOW_ACTION_SET_META;
12797                         break;
12798                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
12799                         if (flow_dv_convert_action_set_tag
12800                                 (dev, mhdr_res,
12801                                  (const struct rte_flow_action_set_tag *)
12802                                   actions->conf, error))
12803                                 return -rte_errno;
12804                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12805                         break;
12806                 case RTE_FLOW_ACTION_TYPE_DROP:
12807                         action_flags |= MLX5_FLOW_ACTION_DROP;
12808                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
12809                         break;
12810                 case RTE_FLOW_ACTION_TYPE_QUEUE:
12811                         queue = actions->conf;
12812                         rss_desc->queue_num = 1;
12813                         rss_desc->queue[0] = queue->index;
12814                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
12815                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
12816                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
12817                         num_of_dest++;
12818                         break;
12819                 case RTE_FLOW_ACTION_TYPE_RSS:
12820                         rss = actions->conf;
12821                         memcpy(rss_desc->queue, rss->queue,
12822                                rss->queue_num * sizeof(uint16_t));
12823                         rss_desc->queue_num = rss->queue_num;
12824                         /* NULL RSS key indicates default RSS key. */
12825                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
12826                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
12827                         /*
12828                          * rss->level and rss.types should be set in advance
12829                          * when expanding items for RSS.
12830                          */
12831                         action_flags |= MLX5_FLOW_ACTION_RSS;
12832                         dev_flow->handle->fate_action = rss_desc->shared_rss ?
12833                                 MLX5_FLOW_FATE_SHARED_RSS :
12834                                 MLX5_FLOW_FATE_QUEUE;
12835                         break;
12836                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
12837                         owner_idx = (uint32_t)(uintptr_t)action->conf;
12838                         age_act = flow_aso_age_get_by_idx(dev, owner_idx);
12839                         if (flow->age == 0) {
12840                                 flow->age = owner_idx;
12841                                 __atomic_fetch_add(&age_act->refcnt, 1,
12842                                                    __ATOMIC_RELAXED);
12843                         }
12844                         age_act_pos = actions_n++;
12845                         action_flags |= MLX5_FLOW_ACTION_AGE;
12846                         break;
12847                 case RTE_FLOW_ACTION_TYPE_AGE:
12848                         non_shared_age = action->conf;
12849                         age_act_pos = actions_n++;
12850                         action_flags |= MLX5_FLOW_ACTION_AGE;
12851                         break;
12852                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
12853                         owner_idx = (uint32_t)(uintptr_t)action->conf;
12854                         cnt_act = flow_dv_counter_get_by_idx(dev, owner_idx,
12855                                                              NULL);
12856                         MLX5_ASSERT(cnt_act != NULL);
12857                         /**
12858                          * When creating meter drop flow in drop table, the
12859                          * counter should not overwrite the rte flow counter.
12860                          */
12861                         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
12862                             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP) {
12863                                 dev_flow->dv.actions[actions_n++] =
12864                                                         cnt_act->action;
12865                         } else {
12866                                 if (flow->counter == 0) {
12867                                         flow->counter = owner_idx;
12868                                         __atomic_fetch_add
12869                                                 (&cnt_act->shared_info.refcnt,
12870                                                  1, __ATOMIC_RELAXED);
12871                                 }
12872                                 /* Save information first, will apply later. */
12873                                 action_flags |= MLX5_FLOW_ACTION_COUNT;
12874                         }
12875                         break;
12876                 case RTE_FLOW_ACTION_TYPE_COUNT:
12877                         if (!dev_conf->devx) {
12878                                 return rte_flow_error_set
12879                                               (error, ENOTSUP,
12880                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12881                                                NULL,
12882                                                "count action not supported");
12883                         }
12884                         /* Save information first, will apply later. */
12885                         count = action->conf;
12886                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12887                         break;
12888                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
12889                         dev_flow->dv.actions[actions_n++] =
12890                                                 priv->sh->pop_vlan_action;
12891                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
12892                         break;
12893                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
12894                         if (!(action_flags &
12895                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
12896                                 flow_dev_get_vlan_info_from_items(items, &vlan);
12897                         vlan.eth_proto = rte_be_to_cpu_16
12898                              ((((const struct rte_flow_action_of_push_vlan *)
12899                                                    actions->conf)->ethertype));
12900                         found_action = mlx5_flow_find_action
12901                                         (actions + 1,
12902                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
12903                         if (found_action)
12904                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12905                         found_action = mlx5_flow_find_action
12906                                         (actions + 1,
12907                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
12908                         if (found_action)
12909                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12910                         if (flow_dv_create_action_push_vlan
12911                                             (dev, attr, &vlan, dev_flow, error))
12912                                 return -rte_errno;
12913                         dev_flow->dv.actions[actions_n++] =
12914                                         dev_flow->dv.push_vlan_res->action;
12915                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
12916                         break;
12917                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
12918                         /* of_vlan_push action handled this action */
12919                         MLX5_ASSERT(action_flags &
12920                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
12921                         break;
12922                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
12923                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
12924                                 break;
12925                         flow_dev_get_vlan_info_from_items(items, &vlan);
12926                         mlx5_update_vlan_vid_pcp(actions, &vlan);
12927                         /* If no VLAN push - this is a modify header action */
12928                         if (flow_dv_convert_action_modify_vlan_vid
12929                                                 (mhdr_res, actions, error))
12930                                 return -rte_errno;
12931                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
12932                         break;
12933                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
12934                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
12935                         if (flow_dv_create_action_l2_encap(dev, actions,
12936                                                            dev_flow,
12937                                                            attr->transfer,
12938                                                            error))
12939                                 return -rte_errno;
12940                         dev_flow->dv.actions[actions_n++] =
12941                                         dev_flow->dv.encap_decap->action;
12942                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12943                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12944                                 sample_act->action_flags |=
12945                                                         MLX5_FLOW_ACTION_ENCAP;
12946                         break;
12947                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
12948                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
12949                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12950                                                            attr->transfer,
12951                                                            error))
12952                                 return -rte_errno;
12953                         dev_flow->dv.actions[actions_n++] =
12954                                         dev_flow->dv.encap_decap->action;
12955                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12956                         break;
12957                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
12958                         /* Handle encap with preceding decap. */
12959                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
12960                                 if (flow_dv_create_action_raw_encap
12961                                         (dev, actions, dev_flow, attr, error))
12962                                         return -rte_errno;
12963                                 dev_flow->dv.actions[actions_n++] =
12964                                         dev_flow->dv.encap_decap->action;
12965                         } else {
12966                                 /* Handle encap without preceding decap. */
12967                                 if (flow_dv_create_action_l2_encap
12968                                     (dev, actions, dev_flow, attr->transfer,
12969                                      error))
12970                                         return -rte_errno;
12971                                 dev_flow->dv.actions[actions_n++] =
12972                                         dev_flow->dv.encap_decap->action;
12973                         }
12974                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12975                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12976                                 sample_act->action_flags |=
12977                                                         MLX5_FLOW_ACTION_ENCAP;
12978                         break;
12979                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
12980                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
12981                                 ;
12982                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
12983                                 if (flow_dv_create_action_l2_decap
12984                                     (dev, dev_flow, attr->transfer, error))
12985                                         return -rte_errno;
12986                                 dev_flow->dv.actions[actions_n++] =
12987                                         dev_flow->dv.encap_decap->action;
12988                         }
12989                         /* If decap is followed by encap, handle it at encap. */
12990                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12991                         break;
12992                 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP:
12993                         dev_flow->dv.actions[actions_n++] =
12994                                 (void *)(uintptr_t)action->conf;
12995                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12996                         break;
12997                 case RTE_FLOW_ACTION_TYPE_JUMP:
12998                         jump_group = ((const struct rte_flow_action_jump *)
12999                                                         action->conf)->group;
13000                         grp_info.std_tbl_fix = 0;
13001                         if (dev_flow->skip_scale &
13002                                 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT))
13003                                 grp_info.skip_scale = 1;
13004                         else
13005                                 grp_info.skip_scale = 0;
13006                         ret = mlx5_flow_group_to_table(dev, tunnel,
13007                                                        jump_group,
13008                                                        &table,
13009                                                        &grp_info, error);
13010                         if (ret)
13011                                 return ret;
13012                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
13013                                                        attr->transfer,
13014                                                        !!dev_flow->external,
13015                                                        tunnel, jump_group, 0,
13016                                                        0, error);
13017                         if (!tbl)
13018                                 return rte_flow_error_set
13019                                                 (error, errno,
13020                                                  RTE_FLOW_ERROR_TYPE_ACTION,
13021                                                  NULL,
13022                                                  "cannot create jump action.");
13023                         if (flow_dv_jump_tbl_resource_register
13024                             (dev, tbl, dev_flow, error)) {
13025                                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
13026                                 return rte_flow_error_set
13027                                                 (error, errno,
13028                                                  RTE_FLOW_ERROR_TYPE_ACTION,
13029                                                  NULL,
13030                                                  "cannot create jump action.");
13031                         }
13032                         dev_flow->dv.actions[actions_n++] =
13033                                         dev_flow->dv.jump->action;
13034                         action_flags |= MLX5_FLOW_ACTION_JUMP;
13035                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
13036                         sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP;
13037                         num_of_dest++;
13038                         break;
13039                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
13040                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
13041                         if (flow_dv_convert_action_modify_mac
13042                                         (mhdr_res, actions, error))
13043                                 return -rte_errno;
13044                         action_flags |= actions->type ==
13045                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
13046                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
13047                                         MLX5_FLOW_ACTION_SET_MAC_DST;
13048                         break;
13049                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
13050                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
13051                         if (flow_dv_convert_action_modify_ipv4
13052                                         (mhdr_res, actions, error))
13053                                 return -rte_errno;
13054                         action_flags |= actions->type ==
13055                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
13056                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
13057                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
13058                         break;
13059                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
13060                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
13061                         if (flow_dv_convert_action_modify_ipv6
13062                                         (mhdr_res, actions, error))
13063                                 return -rte_errno;
13064                         action_flags |= actions->type ==
13065                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
13066                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
13067                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
13068                         break;
13069                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
13070                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
13071                         if (flow_dv_convert_action_modify_tp
13072                                         (mhdr_res, actions, items,
13073                                          &flow_attr, dev_flow, !!(action_flags &
13074                                          MLX5_FLOW_ACTION_DECAP), error))
13075                                 return -rte_errno;
13076                         action_flags |= actions->type ==
13077                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
13078                                         MLX5_FLOW_ACTION_SET_TP_SRC :
13079                                         MLX5_FLOW_ACTION_SET_TP_DST;
13080                         break;
13081                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
13082                         if (flow_dv_convert_action_modify_dec_ttl
13083                                         (mhdr_res, items, &flow_attr, dev_flow,
13084                                          !!(action_flags &
13085                                          MLX5_FLOW_ACTION_DECAP), error))
13086                                 return -rte_errno;
13087                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
13088                         break;
13089                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
13090                         if (flow_dv_convert_action_modify_ttl
13091                                         (mhdr_res, actions, items, &flow_attr,
13092                                          dev_flow, !!(action_flags &
13093                                          MLX5_FLOW_ACTION_DECAP), error))
13094                                 return -rte_errno;
13095                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
13096                         break;
13097                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
13098                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
13099                         if (flow_dv_convert_action_modify_tcp_seq
13100                                         (mhdr_res, actions, error))
13101                                 return -rte_errno;
13102                         action_flags |= actions->type ==
13103                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
13104                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
13105                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
13106                         break;
13107
13108                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
13109                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
13110                         if (flow_dv_convert_action_modify_tcp_ack
13111                                         (mhdr_res, actions, error))
13112                                 return -rte_errno;
13113                         action_flags |= actions->type ==
13114                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
13115                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
13116                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
13117                         break;
13118                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
13119                         if (flow_dv_convert_action_set_reg
13120                                         (mhdr_res, actions, error))
13121                                 return -rte_errno;
13122                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13123                         break;
13124                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
13125                         if (flow_dv_convert_action_copy_mreg
13126                                         (dev, mhdr_res, actions, error))
13127                                 return -rte_errno;
13128                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13129                         break;
13130                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
13131                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
13132                         dev_flow->handle->fate_action =
13133                                         MLX5_FLOW_FATE_DEFAULT_MISS;
13134                         break;
13135                 case RTE_FLOW_ACTION_TYPE_METER:
13136                         if (!wks->fm)
13137                                 return rte_flow_error_set(error, rte_errno,
13138                                         RTE_FLOW_ERROR_TYPE_ACTION,
13139                                         NULL, "Failed to get meter in flow.");
13140                         /* Set the meter action. */
13141                         dev_flow->dv.actions[actions_n++] =
13142                                 wks->fm->meter_action;
13143                         action_flags |= MLX5_FLOW_ACTION_METER;
13144                         break;
13145                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
13146                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
13147                                                               actions, error))
13148                                 return -rte_errno;
13149                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
13150                         break;
13151                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
13152                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
13153                                                               actions, error))
13154                                 return -rte_errno;
13155                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
13156                         break;
13157                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
13158                         sample_act_pos = actions_n;
13159                         sample = (const struct rte_flow_action_sample *)
13160                                  action->conf;
13161                         actions_n++;
13162                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
13163                         /* put encap action into group if work with port id */
13164                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
13165                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
13166                                 sample_act->action_flags |=
13167                                                         MLX5_FLOW_ACTION_ENCAP;
13168                         break;
13169                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
13170                         if (flow_dv_convert_action_modify_field
13171                                         (dev, mhdr_res, actions, attr, error))
13172                                 return -rte_errno;
13173                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
13174                         break;
13175                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
13176                         owner_idx = (uint32_t)(uintptr_t)action->conf;
13177                         ct = flow_aso_ct_get_by_idx(dev, owner_idx);
13178                         if (!ct)
13179                                 return rte_flow_error_set(error, EINVAL,
13180                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13181                                                 NULL,
13182                                                 "Failed to get CT object.");
13183                         if (mlx5_aso_ct_available(priv->sh, ct))
13184                                 return rte_flow_error_set(error, rte_errno,
13185                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13186                                                 NULL,
13187                                                 "CT is unavailable.");
13188                         if (ct->is_original)
13189                                 dev_flow->dv.actions[actions_n] =
13190                                                         ct->dr_action_orig;
13191                         else
13192                                 dev_flow->dv.actions[actions_n] =
13193                                                         ct->dr_action_rply;
13194                         if (flow->ct == 0) {
13195                                 flow->indirect_type =
13196                                                 MLX5_INDIRECT_ACTION_TYPE_CT;
13197                                 flow->ct = owner_idx;
13198                                 __atomic_fetch_add(&ct->refcnt, 1,
13199                                                    __ATOMIC_RELAXED);
13200                         }
13201                         actions_n++;
13202                         action_flags |= MLX5_FLOW_ACTION_CT;
13203                         break;
13204                 case RTE_FLOW_ACTION_TYPE_END:
13205                         actions_end = true;
13206                         if (mhdr_res->actions_num) {
13207                                 /* create modify action if needed. */
13208                                 if (flow_dv_modify_hdr_resource_register
13209                                         (dev, mhdr_res, dev_flow, error))
13210                                         return -rte_errno;
13211                                 dev_flow->dv.actions[modify_action_position] =
13212                                         handle->dvh.modify_hdr->action;
13213                         }
13214                         /*
13215                          * Handle AGE and COUNT action by single HW counter
13216                          * when they are not shared.
13217                          */
13218                         if (action_flags & MLX5_FLOW_ACTION_AGE) {
13219                                 if ((non_shared_age &&
13220                                      count && !count->shared) ||
13221                                     !(priv->sh->flow_hit_aso_en &&
13222                                       (attr->group || attr->transfer))) {
13223                                         /* Creates age by counters. */
13224                                         cnt_act = flow_dv_prepare_counter
13225                                                                 (dev, dev_flow,
13226                                                                  flow, count,
13227                                                                  non_shared_age,
13228                                                                  error);
13229                                         if (!cnt_act)
13230                                                 return -rte_errno;
13231                                         dev_flow->dv.actions[age_act_pos] =
13232                                                                 cnt_act->action;
13233                                         break;
13234                                 }
13235                                 if (!flow->age && non_shared_age) {
13236                                         flow->age = flow_dv_aso_age_alloc
13237                                                                 (dev, error);
13238                                         if (!flow->age)
13239                                                 return -rte_errno;
13240                                         flow_dv_aso_age_params_init
13241                                                     (dev, flow->age,
13242                                                      non_shared_age->context ?
13243                                                      non_shared_age->context :
13244                                                      (void *)(uintptr_t)
13245                                                      (dev_flow->flow_idx),
13246                                                      non_shared_age->timeout);
13247                                 }
13248                                 age_act = flow_aso_age_get_by_idx(dev,
13249                                                                   flow->age);
13250                                 dev_flow->dv.actions[age_act_pos] =
13251                                                              age_act->dr_action;
13252                         }
13253                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
13254                                 /*
13255                                  * Create one count action, to be used
13256                                  * by all sub-flows.
13257                                  */
13258                                 cnt_act = flow_dv_prepare_counter(dev, dev_flow,
13259                                                                   flow, count,
13260                                                                   NULL, error);
13261                                 if (!cnt_act)
13262                                         return -rte_errno;
13263                                 dev_flow->dv.actions[actions_n++] =
13264                                                                 cnt_act->action;
13265                         }
13266                 default:
13267                         break;
13268                 }
13269                 if (mhdr_res->actions_num &&
13270                     modify_action_position == UINT32_MAX)
13271                         modify_action_position = actions_n++;
13272         }
13273         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
13274                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
13275                 int item_type = items->type;
13276
13277                 if (!mlx5_flow_os_item_supported(item_type))
13278                         return rte_flow_error_set(error, ENOTSUP,
13279                                                   RTE_FLOW_ERROR_TYPE_ITEM,
13280                                                   NULL, "item not supported");
13281                 switch (item_type) {
13282                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
13283                         flow_dv_translate_item_port_id
13284                                 (dev, match_mask, match_value, items, attr);
13285                         last_item = MLX5_FLOW_ITEM_PORT_ID;
13286                         break;
13287                 case RTE_FLOW_ITEM_TYPE_ETH:
13288                         flow_dv_translate_item_eth(match_mask, match_value,
13289                                                    items, tunnel,
13290                                                    dev_flow->dv.group);
13291                         matcher.priority = action_flags &
13292                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
13293                                         !dev_flow->external ?
13294                                         MLX5_PRIORITY_MAP_L3 :
13295                                         MLX5_PRIORITY_MAP_L2;
13296                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
13297                                              MLX5_FLOW_LAYER_OUTER_L2;
13298                         break;
13299                 case RTE_FLOW_ITEM_TYPE_VLAN:
13300                         flow_dv_translate_item_vlan(dev_flow,
13301                                                     match_mask, match_value,
13302                                                     items, tunnel,
13303                                                     dev_flow->dv.group);
13304                         matcher.priority = MLX5_PRIORITY_MAP_L2;
13305                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
13306                                               MLX5_FLOW_LAYER_INNER_VLAN) :
13307                                              (MLX5_FLOW_LAYER_OUTER_L2 |
13308                                               MLX5_FLOW_LAYER_OUTER_VLAN);
13309                         break;
13310                 case RTE_FLOW_ITEM_TYPE_IPV4:
13311                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13312                                                   &item_flags, &tunnel);
13313                         flow_dv_translate_item_ipv4(match_mask, match_value,
13314                                                     items, tunnel,
13315                                                     dev_flow->dv.group);
13316                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13317                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
13318                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
13319                         if (items->mask != NULL &&
13320                             ((const struct rte_flow_item_ipv4 *)
13321                              items->mask)->hdr.next_proto_id) {
13322                                 next_protocol =
13323                                         ((const struct rte_flow_item_ipv4 *)
13324                                          (items->spec))->hdr.next_proto_id;
13325                                 next_protocol &=
13326                                         ((const struct rte_flow_item_ipv4 *)
13327                                          (items->mask))->hdr.next_proto_id;
13328                         } else {
13329                                 /* Reset for inner layer. */
13330                                 next_protocol = 0xff;
13331                         }
13332                         break;
13333                 case RTE_FLOW_ITEM_TYPE_IPV6:
13334                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13335                                                   &item_flags, &tunnel);
13336                         flow_dv_translate_item_ipv6(match_mask, match_value,
13337                                                     items, tunnel,
13338                                                     dev_flow->dv.group);
13339                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13340                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
13341                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
13342                         if (items->mask != NULL &&
13343                             ((const struct rte_flow_item_ipv6 *)
13344                              items->mask)->hdr.proto) {
13345                                 next_protocol =
13346                                         ((const struct rte_flow_item_ipv6 *)
13347                                          items->spec)->hdr.proto;
13348                                 next_protocol &=
13349                                         ((const struct rte_flow_item_ipv6 *)
13350                                          items->mask)->hdr.proto;
13351                         } else {
13352                                 /* Reset for inner layer. */
13353                                 next_protocol = 0xff;
13354                         }
13355                         break;
13356                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
13357                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
13358                                                              match_value,
13359                                                              items, tunnel);
13360                         last_item = tunnel ?
13361                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
13362                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
13363                         if (items->mask != NULL &&
13364                             ((const struct rte_flow_item_ipv6_frag_ext *)
13365                              items->mask)->hdr.next_header) {
13366                                 next_protocol =
13367                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13368                                  items->spec)->hdr.next_header;
13369                                 next_protocol &=
13370                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13371                                  items->mask)->hdr.next_header;
13372                         } else {
13373                                 /* Reset for inner layer. */
13374                                 next_protocol = 0xff;
13375                         }
13376                         break;
13377                 case RTE_FLOW_ITEM_TYPE_TCP:
13378                         flow_dv_translate_item_tcp(match_mask, match_value,
13379                                                    items, tunnel);
13380                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13381                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
13382                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
13383                         break;
13384                 case RTE_FLOW_ITEM_TYPE_UDP:
13385                         flow_dv_translate_item_udp(match_mask, match_value,
13386                                                    items, tunnel);
13387                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13388                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
13389                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
13390                         break;
13391                 case RTE_FLOW_ITEM_TYPE_GRE:
13392                         flow_dv_translate_item_gre(match_mask, match_value,
13393                                                    items, tunnel);
13394                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13395                         last_item = MLX5_FLOW_LAYER_GRE;
13396                         break;
13397                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
13398                         flow_dv_translate_item_gre_key(match_mask,
13399                                                        match_value, items);
13400                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
13401                         break;
13402                 case RTE_FLOW_ITEM_TYPE_NVGRE:
13403                         flow_dv_translate_item_nvgre(match_mask, match_value,
13404                                                      items, tunnel);
13405                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13406                         last_item = MLX5_FLOW_LAYER_GRE;
13407                         break;
13408                 case RTE_FLOW_ITEM_TYPE_VXLAN:
13409                         flow_dv_translate_item_vxlan(dev, attr,
13410                                                      match_mask, match_value,
13411                                                      items, tunnel);
13412                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13413                         last_item = MLX5_FLOW_LAYER_VXLAN;
13414                         break;
13415                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
13416                         flow_dv_translate_item_vxlan_gpe(match_mask,
13417                                                          match_value, items,
13418                                                          tunnel);
13419                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13420                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
13421                         break;
13422                 case RTE_FLOW_ITEM_TYPE_GENEVE:
13423                         flow_dv_translate_item_geneve(match_mask, match_value,
13424                                                       items, tunnel);
13425                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13426                         last_item = MLX5_FLOW_LAYER_GENEVE;
13427                         break;
13428                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
13429                         ret = flow_dv_translate_item_geneve_opt(dev, match_mask,
13430                                                           match_value,
13431                                                           items, error);
13432                         if (ret)
13433                                 return rte_flow_error_set(error, -ret,
13434                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13435                                         "cannot create GENEVE TLV option");
13436                         flow->geneve_tlv_option = 1;
13437                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
13438                         break;
13439                 case RTE_FLOW_ITEM_TYPE_MPLS:
13440                         flow_dv_translate_item_mpls(match_mask, match_value,
13441                                                     items, last_item, tunnel);
13442                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13443                         last_item = MLX5_FLOW_LAYER_MPLS;
13444                         break;
13445                 case RTE_FLOW_ITEM_TYPE_MARK:
13446                         flow_dv_translate_item_mark(dev, match_mask,
13447                                                     match_value, items);
13448                         last_item = MLX5_FLOW_ITEM_MARK;
13449                         break;
13450                 case RTE_FLOW_ITEM_TYPE_META:
13451                         flow_dv_translate_item_meta(dev, match_mask,
13452                                                     match_value, attr, items);
13453                         last_item = MLX5_FLOW_ITEM_METADATA;
13454                         break;
13455                 case RTE_FLOW_ITEM_TYPE_ICMP:
13456                         flow_dv_translate_item_icmp(match_mask, match_value,
13457                                                     items, tunnel);
13458                         last_item = MLX5_FLOW_LAYER_ICMP;
13459                         break;
13460                 case RTE_FLOW_ITEM_TYPE_ICMP6:
13461                         flow_dv_translate_item_icmp6(match_mask, match_value,
13462                                                       items, tunnel);
13463                         last_item = MLX5_FLOW_LAYER_ICMP6;
13464                         break;
13465                 case RTE_FLOW_ITEM_TYPE_TAG:
13466                         flow_dv_translate_item_tag(dev, match_mask,
13467                                                    match_value, items);
13468                         last_item = MLX5_FLOW_ITEM_TAG;
13469                         break;
13470                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
13471                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
13472                                                         match_value, items);
13473                         last_item = MLX5_FLOW_ITEM_TAG;
13474                         break;
13475                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
13476                         flow_dv_translate_item_tx_queue(dev, match_mask,
13477                                                         match_value,
13478                                                         items);
13479                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
13480                         break;
13481                 case RTE_FLOW_ITEM_TYPE_GTP:
13482                         flow_dv_translate_item_gtp(match_mask, match_value,
13483                                                    items, tunnel);
13484                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13485                         last_item = MLX5_FLOW_LAYER_GTP;
13486                         break;
13487                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
13488                         ret = flow_dv_translate_item_gtp_psc(match_mask,
13489                                                           match_value,
13490                                                           items);
13491                         if (ret)
13492                                 return rte_flow_error_set(error, -ret,
13493                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13494                                         "cannot create GTP PSC item");
13495                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
13496                         break;
13497                 case RTE_FLOW_ITEM_TYPE_ECPRI:
13498                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
13499                                 /* Create it only the first time to be used. */
13500                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
13501                                 if (ret)
13502                                         return rte_flow_error_set
13503                                                 (error, -ret,
13504                                                 RTE_FLOW_ERROR_TYPE_ITEM,
13505                                                 NULL,
13506                                                 "cannot create eCPRI parser");
13507                         }
13508                         flow_dv_translate_item_ecpri(dev, match_mask,
13509                                                      match_value, items,
13510                                                      last_item);
13511                         /* No other protocol should follow eCPRI layer. */
13512                         last_item = MLX5_FLOW_LAYER_ECPRI;
13513                         break;
13514                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
13515                         flow_dv_translate_item_integrity(match_mask,
13516                                                          match_value,
13517                                                          head_item, items);
13518                         break;
13519                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
13520                         flow_dv_translate_item_aso_ct(dev, match_mask,
13521                                                       match_value, items);
13522                         break;
13523                 default:
13524                         break;
13525                 }
13526                 item_flags |= last_item;
13527         }
13528         /*
13529          * When E-Switch mode is enabled, we have two cases where we need to
13530          * set the source port manually.
13531          * The first one, is in case of Nic steering rule, and the second is
13532          * E-Switch rule where no port_id item was found. In both cases
13533          * the source port is set according the current port in use.
13534          */
13535         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
13536             (priv->representor || priv->master)) {
13537                 if (flow_dv_translate_item_port_id(dev, match_mask,
13538                                                    match_value, NULL, attr))
13539                         return -rte_errno;
13540         }
13541 #ifdef RTE_LIBRTE_MLX5_DEBUG
13542         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
13543                                               dev_flow->dv.value.buf));
13544 #endif
13545         /*
13546          * Layers may be already initialized from prefix flow if this dev_flow
13547          * is the suffix flow.
13548          */
13549         handle->layers |= item_flags;
13550         if (action_flags & MLX5_FLOW_ACTION_RSS)
13551                 flow_dv_hashfields_set(dev_flow, rss_desc);
13552         /* If has RSS action in the sample action, the Sample/Mirror resource
13553          * should be registered after the hash filed be update.
13554          */
13555         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
13556                 ret = flow_dv_translate_action_sample(dev,
13557                                                       sample,
13558                                                       dev_flow, attr,
13559                                                       &num_of_dest,
13560                                                       sample_actions,
13561                                                       &sample_res,
13562                                                       error);
13563                 if (ret < 0)
13564                         return ret;
13565                 ret = flow_dv_create_action_sample(dev,
13566                                                    dev_flow,
13567                                                    num_of_dest,
13568                                                    &sample_res,
13569                                                    &mdest_res,
13570                                                    sample_actions,
13571                                                    action_flags,
13572                                                    error);
13573                 if (ret < 0)
13574                         return rte_flow_error_set
13575                                                 (error, rte_errno,
13576                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13577                                                 NULL,
13578                                                 "cannot create sample action");
13579                 if (num_of_dest > 1) {
13580                         dev_flow->dv.actions[sample_act_pos] =
13581                         dev_flow->dv.dest_array_res->action;
13582                 } else {
13583                         dev_flow->dv.actions[sample_act_pos] =
13584                         dev_flow->dv.sample_res->verbs_action;
13585                 }
13586         }
13587         /*
13588          * For multiple destination (sample action with ratio=1), the encap
13589          * action and port id action will be combined into group action.
13590          * So need remove the original these actions in the flow and only
13591          * use the sample action instead of.
13592          */
13593         if (num_of_dest > 1 &&
13594             (sample_act->dr_port_id_action || sample_act->dr_jump_action)) {
13595                 int i;
13596                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
13597
13598                 for (i = 0; i < actions_n; i++) {
13599                         if ((sample_act->dr_encap_action &&
13600                                 sample_act->dr_encap_action ==
13601                                 dev_flow->dv.actions[i]) ||
13602                                 (sample_act->dr_port_id_action &&
13603                                 sample_act->dr_port_id_action ==
13604                                 dev_flow->dv.actions[i]) ||
13605                                 (sample_act->dr_jump_action &&
13606                                 sample_act->dr_jump_action ==
13607                                 dev_flow->dv.actions[i]))
13608                                 continue;
13609                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
13610                 }
13611                 memcpy((void *)dev_flow->dv.actions,
13612                                 (void *)temp_actions,
13613                                 tmp_actions_n * sizeof(void *));
13614                 actions_n = tmp_actions_n;
13615         }
13616         dev_flow->dv.actions_n = actions_n;
13617         dev_flow->act_flags = action_flags;
13618         if (wks->skip_matcher_reg)
13619                 return 0;
13620         /* Register matcher. */
13621         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
13622                                     matcher.mask.size);
13623         matcher.priority = mlx5_get_matcher_priority(dev, attr,
13624                                         matcher.priority);
13625         /**
13626          * When creating meter drop flow in drop table, using original
13627          * 5-tuple match, the matcher priority should be lower than
13628          * mtr_id matcher.
13629          */
13630         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
13631             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP &&
13632             matcher.priority <= MLX5_REG_BITS)
13633                 matcher.priority += MLX5_REG_BITS;
13634         /* reserved field no needs to be set to 0 here. */
13635         tbl_key.is_fdb = attr->transfer;
13636         tbl_key.is_egress = attr->egress;
13637         tbl_key.level = dev_flow->dv.group;
13638         tbl_key.id = dev_flow->dv.table_id;
13639         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
13640                                      tunnel, attr->group, error))
13641                 return -rte_errno;
13642         return 0;
13643 }
13644
13645 /**
13646  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13647  * and tunnel.
13648  *
13649  * @param[in, out] action
13650  *   Shred RSS action holding hash RX queue objects.
13651  * @param[in] hash_fields
13652  *   Defines combination of packet fields to participate in RX hash.
13653  * @param[in] tunnel
13654  *   Tunnel type
13655  * @param[in] hrxq_idx
13656  *   Hash RX queue index to set.
13657  *
13658  * @return
13659  *   0 on success, otherwise negative errno value.
13660  */
13661 static int
13662 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
13663                               const uint64_t hash_fields,
13664                               uint32_t hrxq_idx)
13665 {
13666         uint32_t *hrxqs = action->hrxq;
13667
13668         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13669         case MLX5_RSS_HASH_IPV4:
13670                 /* fall-through. */
13671         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13672                 /* fall-through. */
13673         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13674                 hrxqs[0] = hrxq_idx;
13675                 return 0;
13676         case MLX5_RSS_HASH_IPV4_TCP:
13677                 /* fall-through. */
13678         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13679                 /* fall-through. */
13680         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13681                 hrxqs[1] = hrxq_idx;
13682                 return 0;
13683         case MLX5_RSS_HASH_IPV4_UDP:
13684                 /* fall-through. */
13685         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13686                 /* fall-through. */
13687         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13688                 hrxqs[2] = hrxq_idx;
13689                 return 0;
13690         case MLX5_RSS_HASH_IPV6:
13691                 /* fall-through. */
13692         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13693                 /* fall-through. */
13694         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13695                 hrxqs[3] = hrxq_idx;
13696                 return 0;
13697         case MLX5_RSS_HASH_IPV6_TCP:
13698                 /* fall-through. */
13699         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13700                 /* fall-through. */
13701         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13702                 hrxqs[4] = hrxq_idx;
13703                 return 0;
13704         case MLX5_RSS_HASH_IPV6_UDP:
13705                 /* fall-through. */
13706         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13707                 /* fall-through. */
13708         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13709                 hrxqs[5] = hrxq_idx;
13710                 return 0;
13711         case MLX5_RSS_HASH_NONE:
13712                 hrxqs[6] = hrxq_idx;
13713                 return 0;
13714         default:
13715                 return -1;
13716         }
13717 }
13718
13719 /**
13720  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13721  * and tunnel.
13722  *
13723  * @param[in] dev
13724  *   Pointer to the Ethernet device structure.
13725  * @param[in] idx
13726  *   Shared RSS action ID holding hash RX queue objects.
13727  * @param[in] hash_fields
13728  *   Defines combination of packet fields to participate in RX hash.
13729  * @param[in] tunnel
13730  *   Tunnel type
13731  *
13732  * @return
13733  *   Valid hash RX queue index, otherwise 0.
13734  */
13735 static uint32_t
13736 __flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
13737                                  const uint64_t hash_fields)
13738 {
13739         struct mlx5_priv *priv = dev->data->dev_private;
13740         struct mlx5_shared_action_rss *shared_rss =
13741             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
13742         const uint32_t *hrxqs = shared_rss->hrxq;
13743
13744         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13745         case MLX5_RSS_HASH_IPV4:
13746                 /* fall-through. */
13747         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13748                 /* fall-through. */
13749         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13750                 return hrxqs[0];
13751         case MLX5_RSS_HASH_IPV4_TCP:
13752                 /* fall-through. */
13753         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13754                 /* fall-through. */
13755         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13756                 return hrxqs[1];
13757         case MLX5_RSS_HASH_IPV4_UDP:
13758                 /* fall-through. */
13759         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13760                 /* fall-through. */
13761         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13762                 return hrxqs[2];
13763         case MLX5_RSS_HASH_IPV6:
13764                 /* fall-through. */
13765         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13766                 /* fall-through. */
13767         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13768                 return hrxqs[3];
13769         case MLX5_RSS_HASH_IPV6_TCP:
13770                 /* fall-through. */
13771         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13772                 /* fall-through. */
13773         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13774                 return hrxqs[4];
13775         case MLX5_RSS_HASH_IPV6_UDP:
13776                 /* fall-through. */
13777         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13778                 /* fall-through. */
13779         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13780                 return hrxqs[5];
13781         case MLX5_RSS_HASH_NONE:
13782                 return hrxqs[6];
13783         default:
13784                 return 0;
13785         }
13786
13787 }
13788
13789 /**
13790  * Apply the flow to the NIC, lock free,
13791  * (mutex should be acquired by caller).
13792  *
13793  * @param[in] dev
13794  *   Pointer to the Ethernet device structure.
13795  * @param[in, out] flow
13796  *   Pointer to flow structure.
13797  * @param[out] error
13798  *   Pointer to error structure.
13799  *
13800  * @return
13801  *   0 on success, a negative errno value otherwise and rte_errno is set.
13802  */
13803 static int
13804 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
13805               struct rte_flow_error *error)
13806 {
13807         struct mlx5_flow_dv_workspace *dv;
13808         struct mlx5_flow_handle *dh;
13809         struct mlx5_flow_handle_dv *dv_h;
13810         struct mlx5_flow *dev_flow;
13811         struct mlx5_priv *priv = dev->data->dev_private;
13812         uint32_t handle_idx;
13813         int n;
13814         int err;
13815         int idx;
13816         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
13817         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
13818         uint8_t misc_mask;
13819
13820         MLX5_ASSERT(wks);
13821         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
13822                 dev_flow = &wks->flows[idx];
13823                 dv = &dev_flow->dv;
13824                 dh = dev_flow->handle;
13825                 dv_h = &dh->dvh;
13826                 n = dv->actions_n;
13827                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
13828                         if (dv->transfer) {
13829                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13830                                 dv->actions[n++] = priv->sh->dr_drop_action;
13831                         } else {
13832 #ifdef HAVE_MLX5DV_DR
13833                                 /* DR supports drop action placeholder. */
13834                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13835                                 dv->actions[n++] = dv->group ?
13836                                         priv->sh->dr_drop_action :
13837                                         priv->root_drop_action;
13838 #else
13839                                 /* For DV we use the explicit drop queue. */
13840                                 MLX5_ASSERT(priv->drop_queue.hrxq);
13841                                 dv->actions[n++] =
13842                                                 priv->drop_queue.hrxq->action;
13843 #endif
13844                         }
13845                 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
13846                            !dv_h->rix_sample && !dv_h->rix_dest_array)) {
13847                         struct mlx5_hrxq *hrxq;
13848                         uint32_t hrxq_idx;
13849
13850                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc,
13851                                                     &hrxq_idx);
13852                         if (!hrxq) {
13853                                 rte_flow_error_set
13854                                         (error, rte_errno,
13855                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13856                                          "cannot get hash queue");
13857                                 goto error;
13858                         }
13859                         dh->rix_hrxq = hrxq_idx;
13860                         dv->actions[n++] = hrxq->action;
13861                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13862                         struct mlx5_hrxq *hrxq = NULL;
13863                         uint32_t hrxq_idx;
13864
13865                         hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev,
13866                                                 rss_desc->shared_rss,
13867                                                 dev_flow->hash_fields);
13868                         if (hrxq_idx)
13869                                 hrxq = mlx5_ipool_get
13870                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
13871                                          hrxq_idx);
13872                         if (!hrxq) {
13873                                 rte_flow_error_set
13874                                         (error, rte_errno,
13875                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13876                                          "cannot get hash queue");
13877                                 goto error;
13878                         }
13879                         dh->rix_srss = rss_desc->shared_rss;
13880                         dv->actions[n++] = hrxq->action;
13881                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
13882                         if (!priv->sh->default_miss_action) {
13883                                 rte_flow_error_set
13884                                         (error, rte_errno,
13885                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13886                                          "default miss action not be created.");
13887                                 goto error;
13888                         }
13889                         dv->actions[n++] = priv->sh->default_miss_action;
13890                 }
13891                 misc_mask = flow_dv_matcher_enable(dv->value.buf);
13892                 __flow_dv_adjust_buf_size(&dv->value.size, misc_mask);
13893                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
13894                                                (void *)&dv->value, n,
13895                                                dv->actions, &dh->drv_flow);
13896                 if (err) {
13897                         rte_flow_error_set
13898                                 (error, errno,
13899                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
13900                                 NULL,
13901                                 (!priv->config.allow_duplicate_pattern &&
13902                                 errno == EEXIST) ?
13903                                 "duplicating pattern is not allowed" :
13904                                 "hardware refuses to create flow");
13905                         goto error;
13906                 }
13907                 if (priv->vmwa_context &&
13908                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
13909                         /*
13910                          * The rule contains the VLAN pattern.
13911                          * For VF we are going to create VLAN
13912                          * interface to make hypervisor set correct
13913                          * e-Switch vport context.
13914                          */
13915                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
13916                 }
13917         }
13918         return 0;
13919 error:
13920         err = rte_errno; /* Save rte_errno before cleanup. */
13921         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
13922                        handle_idx, dh, next) {
13923                 /* hrxq is union, don't clear it if the flag is not set. */
13924                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
13925                         mlx5_hrxq_release(dev, dh->rix_hrxq);
13926                         dh->rix_hrxq = 0;
13927                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13928                         dh->rix_srss = 0;
13929                 }
13930                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
13931                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
13932         }
13933         rte_errno = err; /* Restore rte_errno. */
13934         return -rte_errno;
13935 }
13936
13937 void
13938 flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
13939                           struct mlx5_list_entry *entry)
13940 {
13941         struct mlx5_flow_dv_matcher *resource = container_of(entry,
13942                                                              typeof(*resource),
13943                                                              entry);
13944
13945         claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
13946         mlx5_free(resource);
13947 }
13948
13949 /**
13950  * Release the flow matcher.
13951  *
13952  * @param dev
13953  *   Pointer to Ethernet device.
13954  * @param port_id
13955  *   Index to port ID action resource.
13956  *
13957  * @return
13958  *   1 while a reference on it exists, 0 when freed.
13959  */
13960 static int
13961 flow_dv_matcher_release(struct rte_eth_dev *dev,
13962                         struct mlx5_flow_handle *handle)
13963 {
13964         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
13965         struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl,
13966                                                             typeof(*tbl), tbl);
13967         int ret;
13968
13969         MLX5_ASSERT(matcher->matcher_object);
13970         ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
13971         flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
13972         return ret;
13973 }
13974
13975 void
13976 flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13977 {
13978         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13979         struct mlx5_flow_dv_encap_decap_resource *res =
13980                                        container_of(entry, typeof(*res), entry);
13981
13982         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13983         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
13984 }
13985
13986 /**
13987  * Release an encap/decap resource.
13988  *
13989  * @param dev
13990  *   Pointer to Ethernet device.
13991  * @param encap_decap_idx
13992  *   Index of encap decap resource.
13993  *
13994  * @return
13995  *   1 while a reference on it exists, 0 when freed.
13996  */
13997 static int
13998 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
13999                                      uint32_t encap_decap_idx)
14000 {
14001         struct mlx5_priv *priv = dev->data->dev_private;
14002         struct mlx5_flow_dv_encap_decap_resource *resource;
14003
14004         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
14005                                   encap_decap_idx);
14006         if (!resource)
14007                 return 0;
14008         MLX5_ASSERT(resource->action);
14009         return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
14010 }
14011
14012 /**
14013  * Release an jump to table action resource.
14014  *
14015  * @param dev
14016  *   Pointer to Ethernet device.
14017  * @param rix_jump
14018  *   Index to the jump action resource.
14019  *
14020  * @return
14021  *   1 while a reference on it exists, 0 when freed.
14022  */
14023 static int
14024 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
14025                                   uint32_t rix_jump)
14026 {
14027         struct mlx5_priv *priv = dev->data->dev_private;
14028         struct mlx5_flow_tbl_data_entry *tbl_data;
14029
14030         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
14031                                   rix_jump);
14032         if (!tbl_data)
14033                 return 0;
14034         return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl);
14035 }
14036
14037 void
14038 flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14039 {
14040         struct mlx5_flow_dv_modify_hdr_resource *res =
14041                 container_of(entry, typeof(*res), entry);
14042         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14043
14044         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
14045         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
14046 }
14047
14048 /**
14049  * Release a modify-header resource.
14050  *
14051  * @param dev
14052  *   Pointer to Ethernet device.
14053  * @param handle
14054  *   Pointer to mlx5_flow_handle.
14055  *
14056  * @return
14057  *   1 while a reference on it exists, 0 when freed.
14058  */
14059 static int
14060 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
14061                                     struct mlx5_flow_handle *handle)
14062 {
14063         struct mlx5_priv *priv = dev->data->dev_private;
14064         struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr;
14065
14066         MLX5_ASSERT(entry->action);
14067         return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry);
14068 }
14069
14070 void
14071 flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14072 {
14073         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14074         struct mlx5_flow_dv_port_id_action_resource *resource =
14075                                   container_of(entry, typeof(*resource), entry);
14076
14077         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14078         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
14079 }
14080
14081 /**
14082  * Release port ID action resource.
14083  *
14084  * @param dev
14085  *   Pointer to Ethernet device.
14086  * @param handle
14087  *   Pointer to mlx5_flow_handle.
14088  *
14089  * @return
14090  *   1 while a reference on it exists, 0 when freed.
14091  */
14092 static int
14093 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
14094                                         uint32_t port_id)
14095 {
14096         struct mlx5_priv *priv = dev->data->dev_private;
14097         struct mlx5_flow_dv_port_id_action_resource *resource;
14098
14099         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
14100         if (!resource)
14101                 return 0;
14102         MLX5_ASSERT(resource->action);
14103         return mlx5_list_unregister(priv->sh->port_id_action_list,
14104                                     &resource->entry);
14105 }
14106
14107 /**
14108  * Release shared RSS action resource.
14109  *
14110  * @param dev
14111  *   Pointer to Ethernet device.
14112  * @param srss
14113  *   Shared RSS action index.
14114  */
14115 static void
14116 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
14117 {
14118         struct mlx5_priv *priv = dev->data->dev_private;
14119         struct mlx5_shared_action_rss *shared_rss;
14120
14121         shared_rss = mlx5_ipool_get
14122                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss);
14123         __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14124 }
14125
14126 void
14127 flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14128 {
14129         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14130         struct mlx5_flow_dv_push_vlan_action_resource *resource =
14131                         container_of(entry, typeof(*resource), entry);
14132
14133         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14134         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
14135 }
14136
14137 /**
14138  * Release push vlan action resource.
14139  *
14140  * @param dev
14141  *   Pointer to Ethernet device.
14142  * @param handle
14143  *   Pointer to mlx5_flow_handle.
14144  *
14145  * @return
14146  *   1 while a reference on it exists, 0 when freed.
14147  */
14148 static int
14149 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
14150                                           struct mlx5_flow_handle *handle)
14151 {
14152         struct mlx5_priv *priv = dev->data->dev_private;
14153         struct mlx5_flow_dv_push_vlan_action_resource *resource;
14154         uint32_t idx = handle->dvh.rix_push_vlan;
14155
14156         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
14157         if (!resource)
14158                 return 0;
14159         MLX5_ASSERT(resource->action);
14160         return mlx5_list_unregister(priv->sh->push_vlan_action_list,
14161                                     &resource->entry);
14162 }
14163
14164 /**
14165  * Release the fate resource.
14166  *
14167  * @param dev
14168  *   Pointer to Ethernet device.
14169  * @param handle
14170  *   Pointer to mlx5_flow_handle.
14171  */
14172 static void
14173 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
14174                                struct mlx5_flow_handle *handle)
14175 {
14176         if (!handle->rix_fate)
14177                 return;
14178         switch (handle->fate_action) {
14179         case MLX5_FLOW_FATE_QUEUE:
14180                 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array)
14181                         mlx5_hrxq_release(dev, handle->rix_hrxq);
14182                 break;
14183         case MLX5_FLOW_FATE_JUMP:
14184                 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump);
14185                 break;
14186         case MLX5_FLOW_FATE_PORT_ID:
14187                 flow_dv_port_id_action_resource_release(dev,
14188                                 handle->rix_port_id_action);
14189                 break;
14190         default:
14191                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
14192                 break;
14193         }
14194         handle->rix_fate = 0;
14195 }
14196
14197 void
14198 flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
14199                          struct mlx5_list_entry *entry)
14200 {
14201         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
14202                                                               typeof(*resource),
14203                                                               entry);
14204         struct rte_eth_dev *dev = resource->dev;
14205         struct mlx5_priv *priv = dev->data->dev_private;
14206
14207         if (resource->verbs_action)
14208                 claim_zero(mlx5_flow_os_destroy_flow_action
14209                                                       (resource->verbs_action));
14210         if (resource->normal_path_tbl)
14211                 flow_dv_tbl_resource_release(MLX5_SH(dev),
14212                                              resource->normal_path_tbl);
14213         flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
14214         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
14215         DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
14216 }
14217
14218 /**
14219  * Release an sample resource.
14220  *
14221  * @param dev
14222  *   Pointer to Ethernet device.
14223  * @param handle
14224  *   Pointer to mlx5_flow_handle.
14225  *
14226  * @return
14227  *   1 while a reference on it exists, 0 when freed.
14228  */
14229 static int
14230 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
14231                                      struct mlx5_flow_handle *handle)
14232 {
14233         struct mlx5_priv *priv = dev->data->dev_private;
14234         struct mlx5_flow_dv_sample_resource *resource;
14235
14236         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
14237                                   handle->dvh.rix_sample);
14238         if (!resource)
14239                 return 0;
14240         MLX5_ASSERT(resource->verbs_action);
14241         return mlx5_list_unregister(priv->sh->sample_action_list,
14242                                     &resource->entry);
14243 }
14244
14245 void
14246 flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
14247                              struct mlx5_list_entry *entry)
14248 {
14249         struct mlx5_flow_dv_dest_array_resource *resource =
14250                         container_of(entry, typeof(*resource), entry);
14251         struct rte_eth_dev *dev = resource->dev;
14252         struct mlx5_priv *priv = dev->data->dev_private;
14253         uint32_t i = 0;
14254
14255         MLX5_ASSERT(resource->action);
14256         if (resource->action)
14257                 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14258         for (; i < resource->num_of_dest; i++)
14259                 flow_dv_sample_sub_actions_release(dev,
14260                                                    &resource->sample_idx[i]);
14261         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
14262         DRV_LOG(DEBUG, "destination array resource %p: removed",
14263                 (void *)resource);
14264 }
14265
14266 /**
14267  * Release an destination array resource.
14268  *
14269  * @param dev
14270  *   Pointer to Ethernet device.
14271  * @param handle
14272  *   Pointer to mlx5_flow_handle.
14273  *
14274  * @return
14275  *   1 while a reference on it exists, 0 when freed.
14276  */
14277 static int
14278 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
14279                                     struct mlx5_flow_handle *handle)
14280 {
14281         struct mlx5_priv *priv = dev->data->dev_private;
14282         struct mlx5_flow_dv_dest_array_resource *resource;
14283
14284         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
14285                                   handle->dvh.rix_dest_array);
14286         if (!resource)
14287                 return 0;
14288         MLX5_ASSERT(resource->action);
14289         return mlx5_list_unregister(priv->sh->dest_array_list,
14290                                     &resource->entry);
14291 }
14292
14293 static void
14294 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev)
14295 {
14296         struct mlx5_priv *priv = dev->data->dev_private;
14297         struct mlx5_dev_ctx_shared *sh = priv->sh;
14298         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
14299                                 sh->geneve_tlv_option_resource;
14300         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
14301         if (geneve_opt_resource) {
14302                 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1,
14303                                          __ATOMIC_RELAXED))) {
14304                         claim_zero(mlx5_devx_cmd_destroy
14305                                         (geneve_opt_resource->obj));
14306                         mlx5_free(sh->geneve_tlv_option_resource);
14307                         sh->geneve_tlv_option_resource = NULL;
14308                 }
14309         }
14310         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
14311 }
14312
14313 /**
14314  * Remove the flow from the NIC but keeps it in memory.
14315  * Lock free, (mutex should be acquired by caller).
14316  *
14317  * @param[in] dev
14318  *   Pointer to Ethernet device.
14319  * @param[in, out] flow
14320  *   Pointer to flow structure.
14321  */
14322 static void
14323 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
14324 {
14325         struct mlx5_flow_handle *dh;
14326         uint32_t handle_idx;
14327         struct mlx5_priv *priv = dev->data->dev_private;
14328
14329         if (!flow)
14330                 return;
14331         handle_idx = flow->dev_handles;
14332         while (handle_idx) {
14333                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14334                                     handle_idx);
14335                 if (!dh)
14336                         return;
14337                 if (dh->drv_flow) {
14338                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
14339                         dh->drv_flow = NULL;
14340                 }
14341                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
14342                         flow_dv_fate_resource_release(dev, dh);
14343                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
14344                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
14345                 handle_idx = dh->next.next;
14346         }
14347 }
14348
14349 /**
14350  * Remove the flow from the NIC and the memory.
14351  * Lock free, (mutex should be acquired by caller).
14352  *
14353  * @param[in] dev
14354  *   Pointer to the Ethernet device structure.
14355  * @param[in, out] flow
14356  *   Pointer to flow structure.
14357  */
14358 static void
14359 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
14360 {
14361         struct mlx5_flow_handle *dev_handle;
14362         struct mlx5_priv *priv = dev->data->dev_private;
14363         struct mlx5_flow_meter_info *fm = NULL;
14364         uint32_t srss = 0;
14365
14366         if (!flow)
14367                 return;
14368         flow_dv_remove(dev, flow);
14369         if (flow->counter) {
14370                 flow_dv_counter_free(dev, flow->counter);
14371                 flow->counter = 0;
14372         }
14373         if (flow->meter) {
14374                 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
14375                 if (fm)
14376                         mlx5_flow_meter_detach(priv, fm);
14377                 flow->meter = 0;
14378         }
14379         /* Keep the current age handling by default. */
14380         if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct)
14381                 flow_dv_aso_ct_release(dev, flow->ct, NULL);
14382         else if (flow->age)
14383                 flow_dv_aso_age_release(dev, flow->age);
14384         if (flow->geneve_tlv_option) {
14385                 flow_dv_geneve_tlv_option_resource_release(dev);
14386                 flow->geneve_tlv_option = 0;
14387         }
14388         while (flow->dev_handles) {
14389                 uint32_t tmp_idx = flow->dev_handles;
14390
14391                 dev_handle = mlx5_ipool_get(priv->sh->ipool
14392                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
14393                 if (!dev_handle)
14394                         return;
14395                 flow->dev_handles = dev_handle->next.next;
14396                 if (dev_handle->dvh.matcher)
14397                         flow_dv_matcher_release(dev, dev_handle);
14398                 if (dev_handle->dvh.rix_sample)
14399                         flow_dv_sample_resource_release(dev, dev_handle);
14400                 if (dev_handle->dvh.rix_dest_array)
14401                         flow_dv_dest_array_resource_release(dev, dev_handle);
14402                 if (dev_handle->dvh.rix_encap_decap)
14403                         flow_dv_encap_decap_resource_release(dev,
14404                                 dev_handle->dvh.rix_encap_decap);
14405                 if (dev_handle->dvh.modify_hdr)
14406                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
14407                 if (dev_handle->dvh.rix_push_vlan)
14408                         flow_dv_push_vlan_action_resource_release(dev,
14409                                                                   dev_handle);
14410                 if (dev_handle->dvh.rix_tag)
14411                         flow_dv_tag_release(dev,
14412                                             dev_handle->dvh.rix_tag);
14413                 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS)
14414                         flow_dv_fate_resource_release(dev, dev_handle);
14415                 else if (!srss)
14416                         srss = dev_handle->rix_srss;
14417                 if (fm && dev_handle->is_meter_flow_id &&
14418                     dev_handle->split_flow_id)
14419                         mlx5_ipool_free(fm->flow_ipool,
14420                                         dev_handle->split_flow_id);
14421                 else if (dev_handle->split_flow_id &&
14422                     !dev_handle->is_meter_flow_id)
14423                         mlx5_ipool_free(priv->sh->ipool
14424                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
14425                                         dev_handle->split_flow_id);
14426                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14427                            tmp_idx);
14428         }
14429         if (srss)
14430                 flow_dv_shared_rss_action_release(dev, srss);
14431 }
14432
14433 /**
14434  * Release array of hash RX queue objects.
14435  * Helper function.
14436  *
14437  * @param[in] dev
14438  *   Pointer to the Ethernet device structure.
14439  * @param[in, out] hrxqs
14440  *   Array of hash RX queue objects.
14441  *
14442  * @return
14443  *   Total number of references to hash RX queue objects in *hrxqs* array
14444  *   after this operation.
14445  */
14446 static int
14447 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
14448                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
14449 {
14450         size_t i;
14451         int remaining = 0;
14452
14453         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
14454                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
14455
14456                 if (!ret)
14457                         (*hrxqs)[i] = 0;
14458                 remaining += ret;
14459         }
14460         return remaining;
14461 }
14462
14463 /**
14464  * Release all hash RX queue objects representing shared RSS action.
14465  *
14466  * @param[in] dev
14467  *   Pointer to the Ethernet device structure.
14468  * @param[in, out] action
14469  *   Shared RSS action to remove hash RX queue objects from.
14470  *
14471  * @return
14472  *   Total number of references to hash RX queue objects stored in *action*
14473  *   after this operation.
14474  *   Expected to be 0 if no external references held.
14475  */
14476 static int
14477 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
14478                                  struct mlx5_shared_action_rss *shared_rss)
14479 {
14480         return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq);
14481 }
14482
14483 /**
14484  * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to
14485  * user input.
14486  *
14487  * Only one hash value is available for one L3+L4 combination:
14488  * for example:
14489  * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and
14490  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
14491  * same slot in mlx5_rss_hash_fields.
14492  *
14493  * @param[in] rss
14494  *   Pointer to the shared action RSS conf.
14495  * @param[in, out] hash_field
14496  *   hash_field variable needed to be adjusted.
14497  *
14498  * @return
14499  *   void
14500  */
14501 static void
14502 __flow_dv_action_rss_l34_hash_adjust(struct mlx5_shared_action_rss *rss,
14503                                      uint64_t *hash_field)
14504 {
14505         uint64_t rss_types = rss->origin.types;
14506
14507         switch (*hash_field & ~IBV_RX_HASH_INNER) {
14508         case MLX5_RSS_HASH_IPV4:
14509                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
14510                         *hash_field &= ~MLX5_RSS_HASH_IPV4;
14511                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14512                                 *hash_field |= IBV_RX_HASH_DST_IPV4;
14513                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14514                                 *hash_field |= IBV_RX_HASH_SRC_IPV4;
14515                         else
14516                                 *hash_field |= MLX5_RSS_HASH_IPV4;
14517                 }
14518                 return;
14519         case MLX5_RSS_HASH_IPV6:
14520                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
14521                         *hash_field &= ~MLX5_RSS_HASH_IPV6;
14522                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14523                                 *hash_field |= IBV_RX_HASH_DST_IPV6;
14524                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14525                                 *hash_field |= IBV_RX_HASH_SRC_IPV6;
14526                         else
14527                                 *hash_field |= MLX5_RSS_HASH_IPV6;
14528                 }
14529                 return;
14530         case MLX5_RSS_HASH_IPV4_UDP:
14531                 /* fall-through. */
14532         case MLX5_RSS_HASH_IPV6_UDP:
14533                 if (rss_types & ETH_RSS_UDP) {
14534                         *hash_field &= ~MLX5_UDP_IBV_RX_HASH;
14535                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14536                                 *hash_field |= IBV_RX_HASH_DST_PORT_UDP;
14537                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14538                                 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP;
14539                         else
14540                                 *hash_field |= MLX5_UDP_IBV_RX_HASH;
14541                 }
14542                 return;
14543         case MLX5_RSS_HASH_IPV4_TCP:
14544                 /* fall-through. */
14545         case MLX5_RSS_HASH_IPV6_TCP:
14546                 if (rss_types & ETH_RSS_TCP) {
14547                         *hash_field &= ~MLX5_TCP_IBV_RX_HASH;
14548                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14549                                 *hash_field |= IBV_RX_HASH_DST_PORT_TCP;
14550                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14551                                 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP;
14552                         else
14553                                 *hash_field |= MLX5_TCP_IBV_RX_HASH;
14554                 }
14555                 return;
14556         default:
14557                 return;
14558         }
14559 }
14560
14561 /**
14562  * Setup shared RSS action.
14563  * Prepare set of hash RX queue objects sufficient to handle all valid
14564  * hash_fields combinations (see enum ibv_rx_hash_fields).
14565  *
14566  * @param[in] dev
14567  *   Pointer to the Ethernet device structure.
14568  * @param[in] action_idx
14569  *   Shared RSS action ipool index.
14570  * @param[in, out] action
14571  *   Partially initialized shared RSS action.
14572  * @param[out] error
14573  *   Perform verbose error reporting if not NULL. Initialized in case of
14574  *   error only.
14575  *
14576  * @return
14577  *   0 on success, otherwise negative errno value.
14578  */
14579 static int
14580 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
14581                            uint32_t action_idx,
14582                            struct mlx5_shared_action_rss *shared_rss,
14583                            struct rte_flow_error *error)
14584 {
14585         struct mlx5_flow_rss_desc rss_desc = { 0 };
14586         size_t i;
14587         int err;
14588
14589         if (mlx5_ind_table_obj_setup(dev, shared_rss->ind_tbl)) {
14590                 return rte_flow_error_set(error, rte_errno,
14591                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14592                                           "cannot setup indirection table");
14593         }
14594         memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN);
14595         rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
14596         rss_desc.const_q = shared_rss->origin.queue;
14597         rss_desc.queue_num = shared_rss->origin.queue_num;
14598         /* Set non-zero value to indicate a shared RSS. */
14599         rss_desc.shared_rss = action_idx;
14600         rss_desc.ind_tbl = shared_rss->ind_tbl;
14601         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
14602                 uint32_t hrxq_idx;
14603                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
14604                 int tunnel = 0;
14605
14606                 __flow_dv_action_rss_l34_hash_adjust(shared_rss, &hash_fields);
14607                 if (shared_rss->origin.level > 1) {
14608                         hash_fields |= IBV_RX_HASH_INNER;
14609                         tunnel = 1;
14610                 }
14611                 rss_desc.tunnel = tunnel;
14612                 rss_desc.hash_fields = hash_fields;
14613                 hrxq_idx = mlx5_hrxq_get(dev, &rss_desc);
14614                 if (!hrxq_idx) {
14615                         rte_flow_error_set
14616                                 (error, rte_errno,
14617                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14618                                  "cannot get hash queue");
14619                         goto error_hrxq_new;
14620                 }
14621                 err = __flow_dv_action_rss_hrxq_set
14622                         (shared_rss, hash_fields, hrxq_idx);
14623                 MLX5_ASSERT(!err);
14624         }
14625         return 0;
14626 error_hrxq_new:
14627         err = rte_errno;
14628         __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14629         if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true))
14630                 shared_rss->ind_tbl = NULL;
14631         rte_errno = err;
14632         return -rte_errno;
14633 }
14634
14635 /**
14636  * Create shared RSS action.
14637  *
14638  * @param[in] dev
14639  *   Pointer to the Ethernet device structure.
14640  * @param[in] conf
14641  *   Shared action configuration.
14642  * @param[in] rss
14643  *   RSS action specification used to create shared action.
14644  * @param[out] error
14645  *   Perform verbose error reporting if not NULL. Initialized in case of
14646  *   error only.
14647  *
14648  * @return
14649  *   A valid shared action ID in case of success, 0 otherwise and
14650  *   rte_errno is set.
14651  */
14652 static uint32_t
14653 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
14654                             const struct rte_flow_indir_action_conf *conf,
14655                             const struct rte_flow_action_rss *rss,
14656                             struct rte_flow_error *error)
14657 {
14658         struct mlx5_priv *priv = dev->data->dev_private;
14659         struct mlx5_shared_action_rss *shared_rss = NULL;
14660         void *queue = NULL;
14661         struct rte_flow_action_rss *origin;
14662         const uint8_t *rss_key;
14663         uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
14664         uint32_t idx;
14665
14666         RTE_SET_USED(conf);
14667         queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14668                             0, SOCKET_ID_ANY);
14669         shared_rss = mlx5_ipool_zmalloc
14670                          (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx);
14671         if (!shared_rss || !queue) {
14672                 rte_flow_error_set(error, ENOMEM,
14673                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14674                                    "cannot allocate resource memory");
14675                 goto error_rss_init;
14676         }
14677         if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) {
14678                 rte_flow_error_set(error, E2BIG,
14679                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14680                                    "rss action number out of range");
14681                 goto error_rss_init;
14682         }
14683         shared_rss->ind_tbl = mlx5_malloc(MLX5_MEM_ZERO,
14684                                           sizeof(*shared_rss->ind_tbl),
14685                                           0, SOCKET_ID_ANY);
14686         if (!shared_rss->ind_tbl) {
14687                 rte_flow_error_set(error, ENOMEM,
14688                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14689                                    "cannot allocate resource memory");
14690                 goto error_rss_init;
14691         }
14692         memcpy(queue, rss->queue, queue_size);
14693         shared_rss->ind_tbl->queues = queue;
14694         shared_rss->ind_tbl->queues_n = rss->queue_num;
14695         origin = &shared_rss->origin;
14696         origin->func = rss->func;
14697         origin->level = rss->level;
14698         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
14699         origin->types = !rss->types ? ETH_RSS_IP : rss->types;
14700         /* NULL RSS key indicates default RSS key. */
14701         rss_key = !rss->key ? rss_hash_default_key : rss->key;
14702         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
14703         origin->key = &shared_rss->key[0];
14704         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
14705         origin->queue = queue;
14706         origin->queue_num = rss->queue_num;
14707         if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error))
14708                 goto error_rss_init;
14709         rte_spinlock_init(&shared_rss->action_rss_sl);
14710         __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14711         rte_spinlock_lock(&priv->shared_act_sl);
14712         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14713                      &priv->rss_shared_actions, idx, shared_rss, next);
14714         rte_spinlock_unlock(&priv->shared_act_sl);
14715         return idx;
14716 error_rss_init:
14717         if (shared_rss) {
14718                 if (shared_rss->ind_tbl)
14719                         mlx5_free(shared_rss->ind_tbl);
14720                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14721                                 idx);
14722         }
14723         if (queue)
14724                 mlx5_free(queue);
14725         return 0;
14726 }
14727
14728 /**
14729  * Destroy the shared RSS action.
14730  * Release related hash RX queue objects.
14731  *
14732  * @param[in] dev
14733  *   Pointer to the Ethernet device structure.
14734  * @param[in] idx
14735  *   The shared RSS action object ID to be removed.
14736  * @param[out] error
14737  *   Perform verbose error reporting if not NULL. Initialized in case of
14738  *   error only.
14739  *
14740  * @return
14741  *   0 on success, otherwise negative errno value.
14742  */
14743 static int
14744 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx,
14745                              struct rte_flow_error *error)
14746 {
14747         struct mlx5_priv *priv = dev->data->dev_private;
14748         struct mlx5_shared_action_rss *shared_rss =
14749             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14750         uint32_t old_refcnt = 1;
14751         int remaining;
14752         uint16_t *queue = NULL;
14753
14754         if (!shared_rss)
14755                 return rte_flow_error_set(error, EINVAL,
14756                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14757                                           "invalid shared action");
14758         if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt,
14759                                          0, 0, __ATOMIC_ACQUIRE,
14760                                          __ATOMIC_RELAXED))
14761                 return rte_flow_error_set(error, EBUSY,
14762                                           RTE_FLOW_ERROR_TYPE_ACTION,
14763                                           NULL,
14764                                           "shared rss has references");
14765         remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14766         if (remaining)
14767                 return rte_flow_error_set(error, EBUSY,
14768                                           RTE_FLOW_ERROR_TYPE_ACTION,
14769                                           NULL,
14770                                           "shared rss hrxq has references");
14771         queue = shared_rss->ind_tbl->queues;
14772         remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true);
14773         if (remaining)
14774                 return rte_flow_error_set(error, EBUSY,
14775                                           RTE_FLOW_ERROR_TYPE_ACTION,
14776                                           NULL,
14777                                           "shared rss indirection table has"
14778                                           " references");
14779         mlx5_free(queue);
14780         rte_spinlock_lock(&priv->shared_act_sl);
14781         ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14782                      &priv->rss_shared_actions, idx, shared_rss, next);
14783         rte_spinlock_unlock(&priv->shared_act_sl);
14784         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14785                         idx);
14786         return 0;
14787 }
14788
14789 /**
14790  * Create indirect action, lock free,
14791  * (mutex should be acquired by caller).
14792  * Dispatcher for action type specific call.
14793  *
14794  * @param[in] dev
14795  *   Pointer to the Ethernet device structure.
14796  * @param[in] conf
14797  *   Shared action configuration.
14798  * @param[in] action
14799  *   Action specification used to create indirect action.
14800  * @param[out] error
14801  *   Perform verbose error reporting if not NULL. Initialized in case of
14802  *   error only.
14803  *
14804  * @return
14805  *   A valid shared action handle in case of success, NULL otherwise and
14806  *   rte_errno is set.
14807  */
14808 static struct rte_flow_action_handle *
14809 flow_dv_action_create(struct rte_eth_dev *dev,
14810                       const struct rte_flow_indir_action_conf *conf,
14811                       const struct rte_flow_action *action,
14812                       struct rte_flow_error *err)
14813 {
14814         struct mlx5_priv *priv = dev->data->dev_private;
14815         uint32_t age_idx = 0;
14816         uint32_t idx = 0;
14817         uint32_t ret = 0;
14818
14819         switch (action->type) {
14820         case RTE_FLOW_ACTION_TYPE_RSS:
14821                 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err);
14822                 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS <<
14823                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14824                 break;
14825         case RTE_FLOW_ACTION_TYPE_AGE:
14826                 age_idx = flow_dv_aso_age_alloc(dev, err);
14827                 if (!age_idx) {
14828                         ret = -rte_errno;
14829                         break;
14830                 }
14831                 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
14832                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
14833                 flow_dv_aso_age_params_init(dev, age_idx,
14834                                         ((const struct rte_flow_action_age *)
14835                                                 action->conf)->context ?
14836                                         ((const struct rte_flow_action_age *)
14837                                                 action->conf)->context :
14838                                         (void *)(uintptr_t)idx,
14839                                         ((const struct rte_flow_action_age *)
14840                                                 action->conf)->timeout);
14841                 ret = age_idx;
14842                 break;
14843         case RTE_FLOW_ACTION_TYPE_COUNT:
14844                 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL);
14845                 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT <<
14846                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14847                 break;
14848         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
14849                 ret = flow_dv_translate_create_conntrack(dev, action->conf,
14850                                                          err);
14851                 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret);
14852                 break;
14853         default:
14854                 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
14855                                    NULL, "action type not supported");
14856                 break;
14857         }
14858         return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL;
14859 }
14860
14861 /**
14862  * Destroy the indirect action.
14863  * Release action related resources on the NIC and the memory.
14864  * Lock free, (mutex should be acquired by caller).
14865  * Dispatcher for action type specific call.
14866  *
14867  * @param[in] dev
14868  *   Pointer to the Ethernet device structure.
14869  * @param[in] handle
14870  *   The indirect action object handle to be removed.
14871  * @param[out] error
14872  *   Perform verbose error reporting if not NULL. Initialized in case of
14873  *   error only.
14874  *
14875  * @return
14876  *   0 on success, otherwise negative errno value.
14877  */
14878 static int
14879 flow_dv_action_destroy(struct rte_eth_dev *dev,
14880                        struct rte_flow_action_handle *handle,
14881                        struct rte_flow_error *error)
14882 {
14883         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14884         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14885         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14886         struct mlx5_flow_counter *cnt;
14887         uint32_t no_flow_refcnt = 1;
14888         int ret;
14889
14890         switch (type) {
14891         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14892                 return __flow_dv_action_rss_release(dev, idx, error);
14893         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
14894                 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL);
14895                 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt,
14896                                                  &no_flow_refcnt, 1, false,
14897                                                  __ATOMIC_ACQUIRE,
14898                                                  __ATOMIC_RELAXED))
14899                         return rte_flow_error_set(error, EBUSY,
14900                                                   RTE_FLOW_ERROR_TYPE_ACTION,
14901                                                   NULL,
14902                                                   "Indirect count action has references");
14903                 flow_dv_counter_free(dev, idx);
14904                 return 0;
14905         case MLX5_INDIRECT_ACTION_TYPE_AGE:
14906                 ret = flow_dv_aso_age_release(dev, idx);
14907                 if (ret)
14908                         /*
14909                          * In this case, the last flow has a reference will
14910                          * actually release the age action.
14911                          */
14912                         DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was"
14913                                 " released with references %d.", idx, ret);
14914                 return 0;
14915         case MLX5_INDIRECT_ACTION_TYPE_CT:
14916                 ret = flow_dv_aso_ct_release(dev, idx, error);
14917                 if (ret < 0)
14918                         return ret;
14919                 if (ret > 0)
14920                         DRV_LOG(DEBUG, "Connection tracking object %u still "
14921                                 "has references %d.", idx, ret);
14922                 return 0;
14923         default:
14924                 return rte_flow_error_set(error, ENOTSUP,
14925                                           RTE_FLOW_ERROR_TYPE_ACTION,
14926                                           NULL,
14927                                           "action type not supported");
14928         }
14929 }
14930
14931 /**
14932  * Updates in place shared RSS action configuration.
14933  *
14934  * @param[in] dev
14935  *   Pointer to the Ethernet device structure.
14936  * @param[in] idx
14937  *   The shared RSS action object ID to be updated.
14938  * @param[in] action_conf
14939  *   RSS action specification used to modify *shared_rss*.
14940  * @param[out] error
14941  *   Perform verbose error reporting if not NULL. Initialized in case of
14942  *   error only.
14943  *
14944  * @return
14945  *   0 on success, otherwise negative errno value.
14946  * @note: currently only support update of RSS queues.
14947  */
14948 static int
14949 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx,
14950                             const struct rte_flow_action_rss *action_conf,
14951                             struct rte_flow_error *error)
14952 {
14953         struct mlx5_priv *priv = dev->data->dev_private;
14954         struct mlx5_shared_action_rss *shared_rss =
14955             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14956         int ret = 0;
14957         void *queue = NULL;
14958         uint16_t *queue_old = NULL;
14959         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
14960
14961         if (!shared_rss)
14962                 return rte_flow_error_set(error, EINVAL,
14963                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14964                                           "invalid shared action to update");
14965         if (priv->obj_ops.ind_table_modify == NULL)
14966                 return rte_flow_error_set(error, ENOTSUP,
14967                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14968                                           "cannot modify indirection table");
14969         queue = mlx5_malloc(MLX5_MEM_ZERO,
14970                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14971                             0, SOCKET_ID_ANY);
14972         if (!queue)
14973                 return rte_flow_error_set(error, ENOMEM,
14974                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14975                                           NULL,
14976                                           "cannot allocate resource memory");
14977         memcpy(queue, action_conf->queue, queue_size);
14978         MLX5_ASSERT(shared_rss->ind_tbl);
14979         rte_spinlock_lock(&shared_rss->action_rss_sl);
14980         queue_old = shared_rss->ind_tbl->queues;
14981         ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl,
14982                                         queue, action_conf->queue_num, true);
14983         if (ret) {
14984                 mlx5_free(queue);
14985                 ret = rte_flow_error_set(error, rte_errno,
14986                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14987                                           "cannot update indirection table");
14988         } else {
14989                 mlx5_free(queue_old);
14990                 shared_rss->origin.queue = queue;
14991                 shared_rss->origin.queue_num = action_conf->queue_num;
14992         }
14993         rte_spinlock_unlock(&shared_rss->action_rss_sl);
14994         return ret;
14995 }
14996
14997 /*
14998  * Updates in place conntrack context or direction.
14999  * Context update should be synchronized.
15000  *
15001  * @param[in] dev
15002  *   Pointer to the Ethernet device structure.
15003  * @param[in] idx
15004  *   The conntrack object ID to be updated.
15005  * @param[in] update
15006  *   Pointer to the structure of information to update.
15007  * @param[out] error
15008  *   Perform verbose error reporting if not NULL. Initialized in case of
15009  *   error only.
15010  *
15011  * @return
15012  *   0 on success, otherwise negative errno value.
15013  */
15014 static int
15015 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx,
15016                            const struct rte_flow_modify_conntrack *update,
15017                            struct rte_flow_error *error)
15018 {
15019         struct mlx5_priv *priv = dev->data->dev_private;
15020         struct mlx5_aso_ct_action *ct;
15021         const struct rte_flow_action_conntrack *new_prf;
15022         int ret = 0;
15023         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15024         uint32_t dev_idx;
15025
15026         if (PORT_ID(priv) != owner)
15027                 return rte_flow_error_set(error, EACCES,
15028                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15029                                           NULL,
15030                                           "CT object owned by another port");
15031         dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15032         ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15033         if (!ct->refcnt)
15034                 return rte_flow_error_set(error, ENOMEM,
15035                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15036                                           NULL,
15037                                           "CT object is inactive");
15038         new_prf = &update->new_ct;
15039         if (update->direction)
15040                 ct->is_original = !!new_prf->is_original_dir;
15041         if (update->state) {
15042                 /* Only validate the profile when it needs to be updated. */
15043                 ret = mlx5_validate_action_ct(dev, new_prf, error);
15044                 if (ret)
15045                         return ret;
15046                 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf);
15047                 if (ret)
15048                         return rte_flow_error_set(error, EIO,
15049                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15050                                         NULL,
15051                                         "Failed to send CT context update WQE");
15052                 /* Block until ready or a failure. */
15053                 ret = mlx5_aso_ct_available(priv->sh, ct);
15054                 if (ret)
15055                         rte_flow_error_set(error, rte_errno,
15056                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15057                                            NULL,
15058                                            "Timeout to get the CT update");
15059         }
15060         return ret;
15061 }
15062
15063 /**
15064  * Updates in place shared action configuration, lock free,
15065  * (mutex should be acquired by caller).
15066  *
15067  * @param[in] dev
15068  *   Pointer to the Ethernet device structure.
15069  * @param[in] handle
15070  *   The indirect action object handle to be updated.
15071  * @param[in] update
15072  *   Action specification used to modify the action pointed by *handle*.
15073  *   *update* could be of same type with the action pointed by the *handle*
15074  *   handle argument, or some other structures like a wrapper, depending on
15075  *   the indirect action type.
15076  * @param[out] error
15077  *   Perform verbose error reporting if not NULL. Initialized in case of
15078  *   error only.
15079  *
15080  * @return
15081  *   0 on success, otherwise negative errno value.
15082  */
15083 static int
15084 flow_dv_action_update(struct rte_eth_dev *dev,
15085                         struct rte_flow_action_handle *handle,
15086                         const void *update,
15087                         struct rte_flow_error *err)
15088 {
15089         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15090         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15091         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15092         const void *action_conf;
15093
15094         switch (type) {
15095         case MLX5_INDIRECT_ACTION_TYPE_RSS:
15096                 action_conf = ((const struct rte_flow_action *)update)->conf;
15097                 return __flow_dv_action_rss_update(dev, idx, action_conf, err);
15098         case MLX5_INDIRECT_ACTION_TYPE_CT:
15099                 return __flow_dv_action_ct_update(dev, idx, update, err);
15100         default:
15101                 return rte_flow_error_set(err, ENOTSUP,
15102                                           RTE_FLOW_ERROR_TYPE_ACTION,
15103                                           NULL,
15104                                           "action type update not supported");
15105         }
15106 }
15107
15108 /**
15109  * Destroy the meter sub policy table rules.
15110  * Lock free, (mutex should be acquired by caller).
15111  *
15112  * @param[in] dev
15113  *   Pointer to Ethernet device.
15114  * @param[in] sub_policy
15115  *   Pointer to meter sub policy table.
15116  */
15117 static void
15118 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
15119                              struct mlx5_flow_meter_sub_policy *sub_policy)
15120 {
15121         struct mlx5_priv *priv = dev->data->dev_private;
15122         struct mlx5_flow_tbl_data_entry *tbl;
15123         struct mlx5_flow_meter_policy *policy = sub_policy->main_policy;
15124         struct mlx5_flow_meter_info *next_fm;
15125         struct mlx5_sub_policy_color_rule *color_rule;
15126         void *tmp;
15127         uint32_t i;
15128
15129         for (i = 0; i < RTE_COLORS; i++) {
15130                 next_fm = NULL;
15131                 if (i == RTE_COLOR_GREEN && policy &&
15132                     policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
15133                         next_fm = mlx5_flow_meter_find(priv,
15134                                         policy->act_cnt[i].next_mtr_id, NULL);
15135                 RTE_TAILQ_FOREACH_SAFE(color_rule, &sub_policy->color_rules[i],
15136                                    next_port, tmp) {
15137                         claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
15138                         tbl = container_of(color_rule->matcher->tbl,
15139                                            typeof(*tbl), tbl);
15140                         mlx5_list_unregister(tbl->matchers,
15141                                              &color_rule->matcher->entry);
15142                         TAILQ_REMOVE(&sub_policy->color_rules[i],
15143                                      color_rule, next_port);
15144                         mlx5_free(color_rule);
15145                         if (next_fm)
15146                                 mlx5_flow_meter_detach(priv, next_fm);
15147                 }
15148         }
15149         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15150                 if (sub_policy->rix_hrxq[i]) {
15151                         if (policy && !policy->is_hierarchy)
15152                                 mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]);
15153                         sub_policy->rix_hrxq[i] = 0;
15154                 }
15155                 if (sub_policy->jump_tbl[i]) {
15156                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15157                                                      sub_policy->jump_tbl[i]);
15158                         sub_policy->jump_tbl[i] = NULL;
15159                 }
15160         }
15161         if (sub_policy->tbl_rsc) {
15162                 flow_dv_tbl_resource_release(MLX5_SH(dev),
15163                                              sub_policy->tbl_rsc);
15164                 sub_policy->tbl_rsc = NULL;
15165         }
15166 }
15167
15168 /**
15169  * Destroy policy rules, lock free,
15170  * (mutex should be acquired by caller).
15171  * Dispatcher for action type specific call.
15172  *
15173  * @param[in] dev
15174  *   Pointer to the Ethernet device structure.
15175  * @param[in] mtr_policy
15176  *   Meter policy struct.
15177  */
15178 static void
15179 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev,
15180                              struct mlx5_flow_meter_policy *mtr_policy)
15181 {
15182         uint32_t i, j;
15183         struct mlx5_flow_meter_sub_policy *sub_policy;
15184         uint16_t sub_policy_num;
15185
15186         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15187                 sub_policy_num = (mtr_policy->sub_policy_num >>
15188                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15189                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15190                 for (j = 0; j < sub_policy_num; j++) {
15191                         sub_policy = mtr_policy->sub_policys[i][j];
15192                         if (sub_policy)
15193                                 __flow_dv_destroy_sub_policy_rules(dev,
15194                                                                    sub_policy);
15195                 }
15196         }
15197 }
15198
15199 /**
15200  * Destroy policy action, lock free,
15201  * (mutex should be acquired by caller).
15202  * Dispatcher for action type specific call.
15203  *
15204  * @param[in] dev
15205  *   Pointer to the Ethernet device structure.
15206  * @param[in] mtr_policy
15207  *   Meter policy struct.
15208  */
15209 static void
15210 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
15211                       struct mlx5_flow_meter_policy *mtr_policy)
15212 {
15213         struct rte_flow_action *rss_action;
15214         struct mlx5_flow_handle dev_handle;
15215         uint32_t i, j;
15216
15217         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15218                 if (mtr_policy->act_cnt[i].rix_mark) {
15219                         flow_dv_tag_release(dev,
15220                                 mtr_policy->act_cnt[i].rix_mark);
15221                         mtr_policy->act_cnt[i].rix_mark = 0;
15222                 }
15223                 if (mtr_policy->act_cnt[i].modify_hdr) {
15224                         dev_handle.dvh.modify_hdr =
15225                                 mtr_policy->act_cnt[i].modify_hdr;
15226                         flow_dv_modify_hdr_resource_release(dev, &dev_handle);
15227                 }
15228                 switch (mtr_policy->act_cnt[i].fate_action) {
15229                 case MLX5_FLOW_FATE_SHARED_RSS:
15230                         rss_action = mtr_policy->act_cnt[i].rss;
15231                         mlx5_free(rss_action);
15232                         break;
15233                 case MLX5_FLOW_FATE_PORT_ID:
15234                         if (mtr_policy->act_cnt[i].rix_port_id_action) {
15235                                 flow_dv_port_id_action_resource_release(dev,
15236                                 mtr_policy->act_cnt[i].rix_port_id_action);
15237                                 mtr_policy->act_cnt[i].rix_port_id_action = 0;
15238                         }
15239                         break;
15240                 case MLX5_FLOW_FATE_DROP:
15241                 case MLX5_FLOW_FATE_JUMP:
15242                         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15243                                 mtr_policy->act_cnt[i].dr_jump_action[j] =
15244                                                 NULL;
15245                         break;
15246                 default:
15247                         /*Queue action do nothing*/
15248                         break;
15249                 }
15250         }
15251         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15252                 mtr_policy->dr_drop_action[j] = NULL;
15253 }
15254
15255 /**
15256  * Create policy action per domain, lock free,
15257  * (mutex should be acquired by caller).
15258  * Dispatcher for action type specific call.
15259  *
15260  * @param[in] dev
15261  *   Pointer to the Ethernet device structure.
15262  * @param[in] mtr_policy
15263  *   Meter policy struct.
15264  * @param[in] action
15265  *   Action specification used to create meter actions.
15266  * @param[out] error
15267  *   Perform verbose error reporting if not NULL. Initialized in case of
15268  *   error only.
15269  *
15270  * @return
15271  *   0 on success, otherwise negative errno value.
15272  */
15273 static int
15274 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
15275                         struct mlx5_flow_meter_policy *mtr_policy,
15276                         const struct rte_flow_action *actions[RTE_COLORS],
15277                         enum mlx5_meter_domain domain,
15278                         struct rte_mtr_error *error)
15279 {
15280         struct mlx5_priv *priv = dev->data->dev_private;
15281         struct rte_flow_error flow_err;
15282         const struct rte_flow_action *act;
15283         uint64_t action_flags;
15284         struct mlx5_flow_handle dh;
15285         struct mlx5_flow dev_flow;
15286         struct mlx5_flow_dv_port_id_action_resource port_id_action;
15287         int i, ret;
15288         uint8_t egress, transfer;
15289         struct mlx5_meter_policy_action_container *act_cnt = NULL;
15290         union {
15291                 struct mlx5_flow_dv_modify_hdr_resource res;
15292                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
15293                             sizeof(struct mlx5_modification_cmd) *
15294                             (MLX5_MAX_MODIFY_NUM + 1)];
15295         } mhdr_dummy;
15296         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
15297
15298         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15299         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15300         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
15301         memset(&dev_flow, 0, sizeof(struct mlx5_flow));
15302         memset(&port_id_action, 0,
15303                sizeof(struct mlx5_flow_dv_port_id_action_resource));
15304         memset(mhdr_res, 0, sizeof(*mhdr_res));
15305         mhdr_res->ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
15306                                        (egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
15307                                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
15308         dev_flow.handle = &dh;
15309         dev_flow.dv.port_id_action = &port_id_action;
15310         dev_flow.external = true;
15311         for (i = 0; i < RTE_COLORS; i++) {
15312                 if (i < MLX5_MTR_RTE_COLORS)
15313                         act_cnt = &mtr_policy->act_cnt[i];
15314                 /* Skip the color policy actions creation. */
15315                 if ((i == RTE_COLOR_YELLOW && mtr_policy->skip_y) ||
15316                     (i == RTE_COLOR_GREEN && mtr_policy->skip_g))
15317                         continue;
15318                 action_flags = 0;
15319                 for (act = actions[i];
15320                      act && act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
15321                         switch (act->type) {
15322                         case RTE_FLOW_ACTION_TYPE_MARK:
15323                         {
15324                                 uint32_t tag_be = mlx5_flow_mark_set
15325                                         (((const struct rte_flow_action_mark *)
15326                                         (act->conf))->id);
15327
15328                                 if (i >= MLX5_MTR_RTE_COLORS)
15329                                         return -rte_mtr_error_set(error,
15330                                           ENOTSUP,
15331                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15332                                           NULL,
15333                                           "cannot create policy "
15334                                           "mark action for this color");
15335                                 dev_flow.handle->mark = 1;
15336                                 if (flow_dv_tag_resource_register(dev, tag_be,
15337                                                   &dev_flow, &flow_err))
15338                                         return -rte_mtr_error_set(error,
15339                                         ENOTSUP,
15340                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15341                                         NULL,
15342                                         "cannot setup policy mark action");
15343                                 MLX5_ASSERT(dev_flow.dv.tag_resource);
15344                                 act_cnt->rix_mark =
15345                                         dev_flow.handle->dvh.rix_tag;
15346                                 action_flags |= MLX5_FLOW_ACTION_MARK;
15347                                 break;
15348                         }
15349                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
15350                                 if (i >= MLX5_MTR_RTE_COLORS)
15351                                         return -rte_mtr_error_set(error,
15352                                           ENOTSUP,
15353                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15354                                           NULL,
15355                                           "cannot create policy "
15356                                           "set tag action for this color");
15357                                 if (flow_dv_convert_action_set_tag
15358                                 (dev, mhdr_res,
15359                                 (const struct rte_flow_action_set_tag *)
15360                                 act->conf,  &flow_err))
15361                                         return -rte_mtr_error_set(error,
15362                                         ENOTSUP,
15363                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15364                                         NULL, "cannot convert policy "
15365                                         "set tag action");
15366                                 if (!mhdr_res->actions_num)
15367                                         return -rte_mtr_error_set(error,
15368                                         ENOTSUP,
15369                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15370                                         NULL, "cannot find policy "
15371                                         "set tag action");
15372                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
15373                                 break;
15374                         case RTE_FLOW_ACTION_TYPE_DROP:
15375                         {
15376                                 struct mlx5_flow_mtr_mng *mtrmng =
15377                                                 priv->sh->mtrmng;
15378                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15379
15380                                 /*
15381                                  * Create the drop table with
15382                                  * METER DROP level.
15383                                  */
15384                                 if (!mtrmng->drop_tbl[domain]) {
15385                                         mtrmng->drop_tbl[domain] =
15386                                         flow_dv_tbl_resource_get(dev,
15387                                         MLX5_FLOW_TABLE_LEVEL_METER,
15388                                         egress, transfer, false, NULL, 0,
15389                                         0, MLX5_MTR_TABLE_ID_DROP, &flow_err);
15390                                         if (!mtrmng->drop_tbl[domain])
15391                                                 return -rte_mtr_error_set
15392                                         (error, ENOTSUP,
15393                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15394                                         NULL,
15395                                         "Failed to create meter drop table");
15396                                 }
15397                                 tbl_data = container_of
15398                                 (mtrmng->drop_tbl[domain],
15399                                 struct mlx5_flow_tbl_data_entry, tbl);
15400                                 if (i < MLX5_MTR_RTE_COLORS) {
15401                                         act_cnt->dr_jump_action[domain] =
15402                                                 tbl_data->jump.action;
15403                                         act_cnt->fate_action =
15404                                                 MLX5_FLOW_FATE_DROP;
15405                                 }
15406                                 if (i == RTE_COLOR_RED)
15407                                         mtr_policy->dr_drop_action[domain] =
15408                                                 tbl_data->jump.action;
15409                                 action_flags |= MLX5_FLOW_ACTION_DROP;
15410                                 break;
15411                         }
15412                         case RTE_FLOW_ACTION_TYPE_QUEUE:
15413                         {
15414                                 if (i >= MLX5_MTR_RTE_COLORS)
15415                                         return -rte_mtr_error_set(error,
15416                                         ENOTSUP,
15417                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15418                                         NULL, "cannot create policy "
15419                                         "fate queue for this color");
15420                                 act_cnt->queue =
15421                                 ((const struct rte_flow_action_queue *)
15422                                         (act->conf))->index;
15423                                 act_cnt->fate_action =
15424                                         MLX5_FLOW_FATE_QUEUE;
15425                                 dev_flow.handle->fate_action =
15426                                         MLX5_FLOW_FATE_QUEUE;
15427                                 mtr_policy->is_queue = 1;
15428                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
15429                                 break;
15430                         }
15431                         case RTE_FLOW_ACTION_TYPE_RSS:
15432                         {
15433                                 int rss_size;
15434
15435                                 if (i >= MLX5_MTR_RTE_COLORS)
15436                                         return -rte_mtr_error_set(error,
15437                                           ENOTSUP,
15438                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15439                                           NULL,
15440                                           "cannot create policy "
15441                                           "rss action for this color");
15442                                 /*
15443                                  * Save RSS conf into policy struct
15444                                  * for translate stage.
15445                                  */
15446                                 rss_size = (int)rte_flow_conv
15447                                         (RTE_FLOW_CONV_OP_ACTION,
15448                                         NULL, 0, act, &flow_err);
15449                                 if (rss_size <= 0)
15450                                         return -rte_mtr_error_set(error,
15451                                           ENOTSUP,
15452                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15453                                           NULL, "Get the wrong "
15454                                           "rss action struct size");
15455                                 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO,
15456                                                 rss_size, 0, SOCKET_ID_ANY);
15457                                 if (!act_cnt->rss)
15458                                         return -rte_mtr_error_set(error,
15459                                           ENOTSUP,
15460                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15461                                           NULL,
15462                                           "Fail to malloc rss action memory");
15463                                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION,
15464                                         act_cnt->rss, rss_size,
15465                                         act, &flow_err);
15466                                 if (ret < 0)
15467                                         return -rte_mtr_error_set(error,
15468                                           ENOTSUP,
15469                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15470                                           NULL, "Fail to save "
15471                                           "rss action into policy struct");
15472                                 act_cnt->fate_action =
15473                                         MLX5_FLOW_FATE_SHARED_RSS;
15474                                 action_flags |= MLX5_FLOW_ACTION_RSS;
15475                                 break;
15476                         }
15477                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
15478                         {
15479                                 struct mlx5_flow_dv_port_id_action_resource
15480                                         port_id_resource;
15481                                 uint32_t port_id = 0;
15482
15483                                 if (i >= MLX5_MTR_RTE_COLORS)
15484                                         return -rte_mtr_error_set(error,
15485                                         ENOTSUP,
15486                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15487                                         NULL, "cannot create policy "
15488                                         "port action for this color");
15489                                 memset(&port_id_resource, 0,
15490                                         sizeof(port_id_resource));
15491                                 if (flow_dv_translate_action_port_id(dev, act,
15492                                                 &port_id, &flow_err))
15493                                         return -rte_mtr_error_set(error,
15494                                         ENOTSUP,
15495                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15496                                         NULL, "cannot translate "
15497                                         "policy port action");
15498                                 port_id_resource.port_id = port_id;
15499                                 if (flow_dv_port_id_action_resource_register
15500                                         (dev, &port_id_resource,
15501                                         &dev_flow, &flow_err))
15502                                         return -rte_mtr_error_set(error,
15503                                         ENOTSUP,
15504                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15505                                         NULL, "cannot setup "
15506                                         "policy port action");
15507                                 act_cnt->rix_port_id_action =
15508                                         dev_flow.handle->rix_port_id_action;
15509                                 act_cnt->fate_action =
15510                                         MLX5_FLOW_FATE_PORT_ID;
15511                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
15512                                 break;
15513                         }
15514                         case RTE_FLOW_ACTION_TYPE_JUMP:
15515                         {
15516                                 uint32_t jump_group = 0;
15517                                 uint32_t table = 0;
15518                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15519                                 struct flow_grp_info grp_info = {
15520                                         .external = !!dev_flow.external,
15521                                         .transfer = !!transfer,
15522                                         .fdb_def_rule = !!priv->fdb_def_rule,
15523                                         .std_tbl_fix = 0,
15524                                         .skip_scale = dev_flow.skip_scale &
15525                                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
15526                                 };
15527                                 struct mlx5_flow_meter_sub_policy *sub_policy =
15528                                         mtr_policy->sub_policys[domain][0];
15529
15530                                 if (i >= MLX5_MTR_RTE_COLORS)
15531                                         return -rte_mtr_error_set(error,
15532                                           ENOTSUP,
15533                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15534                                           NULL,
15535                                           "cannot create policy "
15536                                           "jump action for this color");
15537                                 jump_group =
15538                                 ((const struct rte_flow_action_jump *)
15539                                                         act->conf)->group;
15540                                 if (mlx5_flow_group_to_table(dev, NULL,
15541                                                        jump_group,
15542                                                        &table,
15543                                                        &grp_info, &flow_err))
15544                                         return -rte_mtr_error_set(error,
15545                                         ENOTSUP,
15546                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15547                                         NULL, "cannot setup "
15548                                         "policy jump action");
15549                                 sub_policy->jump_tbl[i] =
15550                                 flow_dv_tbl_resource_get(dev,
15551                                         table, egress,
15552                                         transfer,
15553                                         !!dev_flow.external,
15554                                         NULL, jump_group, 0,
15555                                         0, &flow_err);
15556                                 if
15557                                 (!sub_policy->jump_tbl[i])
15558                                         return  -rte_mtr_error_set(error,
15559                                         ENOTSUP,
15560                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15561                                         NULL, "cannot create jump action.");
15562                                 tbl_data = container_of
15563                                 (sub_policy->jump_tbl[i],
15564                                 struct mlx5_flow_tbl_data_entry, tbl);
15565                                 act_cnt->dr_jump_action[domain] =
15566                                         tbl_data->jump.action;
15567                                 act_cnt->fate_action =
15568                                         MLX5_FLOW_FATE_JUMP;
15569                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
15570                                 break;
15571                         }
15572                         /*
15573                          * No need to check meter hierarchy for Y or R colors
15574                          * here since it is done in the validation stage.
15575                          */
15576                         case RTE_FLOW_ACTION_TYPE_METER:
15577                         {
15578                                 const struct rte_flow_action_meter *mtr;
15579                                 struct mlx5_flow_meter_info *next_fm;
15580                                 struct mlx5_flow_meter_policy *next_policy;
15581                                 struct rte_flow_action tag_action;
15582                                 struct mlx5_rte_flow_action_set_tag set_tag;
15583                                 uint32_t next_mtr_idx = 0;
15584
15585                                 mtr = act->conf;
15586                                 next_fm = mlx5_flow_meter_find(priv,
15587                                                         mtr->mtr_id,
15588                                                         &next_mtr_idx);
15589                                 if (!next_fm)
15590                                         return -rte_mtr_error_set(error, EINVAL,
15591                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15592                                                 "Fail to find next meter.");
15593                                 if (next_fm->def_policy)
15594                                         return -rte_mtr_error_set(error, EINVAL,
15595                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15596                                 "Hierarchy only supports termination meter.");
15597                                 next_policy = mlx5_flow_meter_policy_find(dev,
15598                                                 next_fm->policy_id, NULL);
15599                                 MLX5_ASSERT(next_policy);
15600                                 if (next_fm->drop_cnt) {
15601                                         set_tag.id =
15602                                                 (enum modify_reg)
15603                                                 mlx5_flow_get_reg_id(dev,
15604                                                 MLX5_MTR_ID,
15605                                                 0,
15606                                                 (struct rte_flow_error *)error);
15607                                         set_tag.offset = (priv->mtr_reg_share ?
15608                                                 MLX5_MTR_COLOR_BITS : 0);
15609                                         set_tag.length = (priv->mtr_reg_share ?
15610                                                MLX5_MTR_IDLE_BITS_IN_COLOR_REG :
15611                                                MLX5_REG_BITS);
15612                                         set_tag.data = next_mtr_idx;
15613                                         tag_action.type =
15614                                                 (enum rte_flow_action_type)
15615                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
15616                                         tag_action.conf = &set_tag;
15617                                         if (flow_dv_convert_action_set_reg
15618                                                 (mhdr_res, &tag_action,
15619                                                 (struct rte_flow_error *)error))
15620                                                 return -rte_errno;
15621                                         action_flags |=
15622                                                 MLX5_FLOW_ACTION_SET_TAG;
15623                                 }
15624                                 act_cnt->fate_action = MLX5_FLOW_FATE_MTR;
15625                                 act_cnt->next_mtr_id = next_fm->meter_id;
15626                                 act_cnt->next_sub_policy = NULL;
15627                                 mtr_policy->is_hierarchy = 1;
15628                                 mtr_policy->dev = next_policy->dev;
15629                                 action_flags |=
15630                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
15631                                 break;
15632                         }
15633                         default:
15634                                 return -rte_mtr_error_set(error, ENOTSUP,
15635                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15636                                           NULL, "action type not supported");
15637                         }
15638                         if (action_flags & MLX5_FLOW_ACTION_SET_TAG) {
15639                                 /* create modify action if needed. */
15640                                 dev_flow.dv.group = 1;
15641                                 if (flow_dv_modify_hdr_resource_register
15642                                         (dev, mhdr_res, &dev_flow, &flow_err))
15643                                         return -rte_mtr_error_set(error,
15644                                                 ENOTSUP,
15645                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
15646                                                 NULL, "cannot register policy "
15647                                                 "set tag action");
15648                                 act_cnt->modify_hdr =
15649                                         dev_flow.handle->dvh.modify_hdr;
15650                         }
15651                 }
15652         }
15653         return 0;
15654 }
15655
15656 /**
15657  * Create policy action per domain, lock free,
15658  * (mutex should be acquired by caller).
15659  * Dispatcher for action type specific call.
15660  *
15661  * @param[in] dev
15662  *   Pointer to the Ethernet device structure.
15663  * @param[in] mtr_policy
15664  *   Meter policy struct.
15665  * @param[in] action
15666  *   Action specification used to create meter actions.
15667  * @param[out] error
15668  *   Perform verbose error reporting if not NULL. Initialized in case of
15669  *   error only.
15670  *
15671  * @return
15672  *   0 on success, otherwise negative errno value.
15673  */
15674 static int
15675 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev,
15676                       struct mlx5_flow_meter_policy *mtr_policy,
15677                       const struct rte_flow_action *actions[RTE_COLORS],
15678                       struct rte_mtr_error *error)
15679 {
15680         int ret, i;
15681         uint16_t sub_policy_num;
15682
15683         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15684                 sub_policy_num = (mtr_policy->sub_policy_num >>
15685                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15686                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15687                 if (sub_policy_num) {
15688                         ret = __flow_dv_create_domain_policy_acts(dev,
15689                                 mtr_policy, actions,
15690                                 (enum mlx5_meter_domain)i, error);
15691                         /* Cleaning resource is done in the caller level. */
15692                         if (ret)
15693                                 return ret;
15694                 }
15695         }
15696         return 0;
15697 }
15698
15699 /**
15700  * Query a DV flow rule for its statistics via DevX.
15701  *
15702  * @param[in] dev
15703  *   Pointer to Ethernet device.
15704  * @param[in] cnt_idx
15705  *   Index to the flow counter.
15706  * @param[out] data
15707  *   Data retrieved by the query.
15708  * @param[out] error
15709  *   Perform verbose error reporting if not NULL.
15710  *
15711  * @return
15712  *   0 on success, a negative errno value otherwise and rte_errno is set.
15713  */
15714 static int
15715 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data,
15716                     struct rte_flow_error *error)
15717 {
15718         struct mlx5_priv *priv = dev->data->dev_private;
15719         struct rte_flow_query_count *qc = data;
15720
15721         if (!priv->config.devx)
15722                 return rte_flow_error_set(error, ENOTSUP,
15723                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15724                                           NULL,
15725                                           "counters are not supported");
15726         if (cnt_idx) {
15727                 uint64_t pkts, bytes;
15728                 struct mlx5_flow_counter *cnt;
15729                 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes);
15730
15731                 if (err)
15732                         return rte_flow_error_set(error, -err,
15733                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15734                                         NULL, "cannot read counters");
15735                 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL);
15736                 qc->hits_set = 1;
15737                 qc->bytes_set = 1;
15738                 qc->hits = pkts - cnt->hits;
15739                 qc->bytes = bytes - cnt->bytes;
15740                 if (qc->reset) {
15741                         cnt->hits = pkts;
15742                         cnt->bytes = bytes;
15743                 }
15744                 return 0;
15745         }
15746         return rte_flow_error_set(error, EINVAL,
15747                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15748                                   NULL,
15749                                   "counters are not available");
15750 }
15751
15752 static int
15753 flow_dv_action_query(struct rte_eth_dev *dev,
15754                      const struct rte_flow_action_handle *handle, void *data,
15755                      struct rte_flow_error *error)
15756 {
15757         struct mlx5_age_param *age_param;
15758         struct rte_flow_query_age *resp;
15759         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15760         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15761         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15762         struct mlx5_priv *priv = dev->data->dev_private;
15763         struct mlx5_aso_ct_action *ct;
15764         uint16_t owner;
15765         uint32_t dev_idx;
15766
15767         switch (type) {
15768         case MLX5_INDIRECT_ACTION_TYPE_AGE:
15769                 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params;
15770                 resp = data;
15771                 resp->aged = __atomic_load_n(&age_param->state,
15772                                               __ATOMIC_RELAXED) == AGE_TMOUT ?
15773                                                                           1 : 0;
15774                 resp->sec_since_last_hit_valid = !resp->aged;
15775                 if (resp->sec_since_last_hit_valid)
15776                         resp->sec_since_last_hit = __atomic_load_n
15777                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15778                 return 0;
15779         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
15780                 return flow_dv_query_count(dev, idx, data, error);
15781         case MLX5_INDIRECT_ACTION_TYPE_CT:
15782                 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15783                 if (owner != PORT_ID(priv))
15784                         return rte_flow_error_set(error, EACCES,
15785                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15786                                         NULL,
15787                                         "CT object owned by another port");
15788                 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15789                 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15790                 MLX5_ASSERT(ct);
15791                 if (!ct->refcnt)
15792                         return rte_flow_error_set(error, EFAULT,
15793                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15794                                         NULL,
15795                                         "CT object is inactive");
15796                 ((struct rte_flow_action_conntrack *)data)->peer_port =
15797                                                         ct->peer;
15798                 ((struct rte_flow_action_conntrack *)data)->is_original_dir =
15799                                                         ct->is_original;
15800                 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data))
15801                         return rte_flow_error_set(error, EIO,
15802                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15803                                         NULL,
15804                                         "Failed to query CT context");
15805                 return 0;
15806         default:
15807                 return rte_flow_error_set(error, ENOTSUP,
15808                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15809                                           "action type query not supported");
15810         }
15811 }
15812
15813 /**
15814  * Query a flow rule AGE action for aging information.
15815  *
15816  * @param[in] dev
15817  *   Pointer to Ethernet device.
15818  * @param[in] flow
15819  *   Pointer to the sub flow.
15820  * @param[out] data
15821  *   data retrieved by the query.
15822  * @param[out] error
15823  *   Perform verbose error reporting if not NULL.
15824  *
15825  * @return
15826  *   0 on success, a negative errno value otherwise and rte_errno is set.
15827  */
15828 static int
15829 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
15830                   void *data, struct rte_flow_error *error)
15831 {
15832         struct rte_flow_query_age *resp = data;
15833         struct mlx5_age_param *age_param;
15834
15835         if (flow->age) {
15836                 struct mlx5_aso_age_action *act =
15837                                      flow_aso_age_get_by_idx(dev, flow->age);
15838
15839                 age_param = &act->age_params;
15840         } else if (flow->counter) {
15841                 age_param = flow_dv_counter_idx_get_age(dev, flow->counter);
15842
15843                 if (!age_param || !age_param->timeout)
15844                         return rte_flow_error_set
15845                                         (error, EINVAL,
15846                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15847                                          NULL, "cannot read age data");
15848         } else {
15849                 return rte_flow_error_set(error, EINVAL,
15850                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15851                                           NULL, "age data not available");
15852         }
15853         resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
15854                                      AGE_TMOUT ? 1 : 0;
15855         resp->sec_since_last_hit_valid = !resp->aged;
15856         if (resp->sec_since_last_hit_valid)
15857                 resp->sec_since_last_hit = __atomic_load_n
15858                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15859         return 0;
15860 }
15861
15862 /**
15863  * Query a flow.
15864  *
15865  * @see rte_flow_query()
15866  * @see rte_flow_ops
15867  */
15868 static int
15869 flow_dv_query(struct rte_eth_dev *dev,
15870               struct rte_flow *flow __rte_unused,
15871               const struct rte_flow_action *actions __rte_unused,
15872               void *data __rte_unused,
15873               struct rte_flow_error *error __rte_unused)
15874 {
15875         int ret = -EINVAL;
15876
15877         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
15878                 switch (actions->type) {
15879                 case RTE_FLOW_ACTION_TYPE_VOID:
15880                         break;
15881                 case RTE_FLOW_ACTION_TYPE_COUNT:
15882                         ret = flow_dv_query_count(dev, flow->counter, data,
15883                                                   error);
15884                         break;
15885                 case RTE_FLOW_ACTION_TYPE_AGE:
15886                         ret = flow_dv_query_age(dev, flow, data, error);
15887                         break;
15888                 default:
15889                         return rte_flow_error_set(error, ENOTSUP,
15890                                                   RTE_FLOW_ERROR_TYPE_ACTION,
15891                                                   actions,
15892                                                   "action not supported");
15893                 }
15894         }
15895         return ret;
15896 }
15897
15898 /**
15899  * Destroy the meter table set.
15900  * Lock free, (mutex should be acquired by caller).
15901  *
15902  * @param[in] dev
15903  *   Pointer to Ethernet device.
15904  * @param[in] fm
15905  *   Meter information table.
15906  */
15907 static void
15908 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
15909                         struct mlx5_flow_meter_info *fm)
15910 {
15911         struct mlx5_priv *priv = dev->data->dev_private;
15912         int i;
15913
15914         if (!fm || !priv->config.dv_flow_en)
15915                 return;
15916         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15917                 if (fm->drop_rule[i]) {
15918                         claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
15919                         fm->drop_rule[i] = NULL;
15920                 }
15921         }
15922 }
15923
15924 static void
15925 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
15926 {
15927         struct mlx5_priv *priv = dev->data->dev_private;
15928         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15929         struct mlx5_flow_tbl_data_entry *tbl;
15930         int i, j;
15931
15932         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15933                 if (mtrmng->def_rule[i]) {
15934                         claim_zero(mlx5_flow_os_destroy_flow
15935                                         (mtrmng->def_rule[i]));
15936                         mtrmng->def_rule[i] = NULL;
15937                 }
15938                 if (mtrmng->def_matcher[i]) {
15939                         tbl = container_of(mtrmng->def_matcher[i]->tbl,
15940                                 struct mlx5_flow_tbl_data_entry, tbl);
15941                         mlx5_list_unregister(tbl->matchers,
15942                                              &mtrmng->def_matcher[i]->entry);
15943                         mtrmng->def_matcher[i] = NULL;
15944                 }
15945                 for (j = 0; j < MLX5_REG_BITS; j++) {
15946                         if (mtrmng->drop_matcher[i][j]) {
15947                                 tbl =
15948                                 container_of(mtrmng->drop_matcher[i][j]->tbl,
15949                                              struct mlx5_flow_tbl_data_entry,
15950                                              tbl);
15951                                 mlx5_list_unregister(tbl->matchers,
15952                                             &mtrmng->drop_matcher[i][j]->entry);
15953                                 mtrmng->drop_matcher[i][j] = NULL;
15954                         }
15955                 }
15956                 if (mtrmng->drop_tbl[i]) {
15957                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15958                                 mtrmng->drop_tbl[i]);
15959                         mtrmng->drop_tbl[i] = NULL;
15960                 }
15961         }
15962 }
15963
15964 /* Number of meter flow actions, count and jump or count and drop. */
15965 #define METER_ACTIONS 2
15966
15967 static void
15968 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
15969                                     enum mlx5_meter_domain domain)
15970 {
15971         struct mlx5_priv *priv = dev->data->dev_private;
15972         struct mlx5_flow_meter_def_policy *def_policy =
15973                         priv->sh->mtrmng->def_policy[domain];
15974
15975         __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
15976         mlx5_free(def_policy);
15977         priv->sh->mtrmng->def_policy[domain] = NULL;
15978 }
15979
15980 /**
15981  * Destroy the default policy table set.
15982  *
15983  * @param[in] dev
15984  *   Pointer to Ethernet device.
15985  */
15986 static void
15987 flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
15988 {
15989         struct mlx5_priv *priv = dev->data->dev_private;
15990         int i;
15991
15992         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
15993                 if (priv->sh->mtrmng->def_policy[i])
15994                         __flow_dv_destroy_domain_def_policy(dev,
15995                                         (enum mlx5_meter_domain)i);
15996         priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
15997 }
15998
15999 static int
16000 __flow_dv_create_policy_flow(struct rte_eth_dev *dev,
16001                         uint32_t color_reg_c_idx,
16002                         enum rte_color color, void *matcher_object,
16003                         int actions_n, void *actions,
16004                         bool match_src_port, const struct rte_flow_item *item,
16005                         void **rule, const struct rte_flow_attr *attr)
16006 {
16007         int ret;
16008         struct mlx5_flow_dv_match_params value = {
16009                 .size = sizeof(value.buf),
16010         };
16011         struct mlx5_flow_dv_match_params matcher = {
16012                 .size = sizeof(matcher.buf),
16013         };
16014         struct mlx5_priv *priv = dev->data->dev_private;
16015         uint8_t misc_mask;
16016
16017         if (match_src_port && (priv->representor || priv->master)) {
16018                 if (flow_dv_translate_item_port_id(dev, matcher.buf,
16019                                                    value.buf, item, attr)) {
16020                         DRV_LOG(ERR, "Failed to create meter policy%d flow's"
16021                                 " value with port.", color);
16022                         return -1;
16023                 }
16024         }
16025         flow_dv_match_meta_reg(matcher.buf, value.buf,
16026                                (enum modify_reg)color_reg_c_idx,
16027                                rte_col_2_mlx5_col(color), UINT32_MAX);
16028         misc_mask = flow_dv_matcher_enable(value.buf);
16029         __flow_dv_adjust_buf_size(&value.size, misc_mask);
16030         ret = mlx5_flow_os_create_flow(matcher_object, (void *)&value,
16031                                        actions_n, actions, rule);
16032         if (ret) {
16033                 DRV_LOG(ERR, "Failed to create meter policy%d flow.", color);
16034                 return -1;
16035         }
16036         return 0;
16037 }
16038
16039 static int
16040 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
16041                         uint32_t color_reg_c_idx,
16042                         uint16_t priority,
16043                         struct mlx5_flow_meter_sub_policy *sub_policy,
16044                         const struct rte_flow_attr *attr,
16045                         bool match_src_port,
16046                         const struct rte_flow_item *item,
16047                         struct mlx5_flow_dv_matcher **policy_matcher,
16048                         struct rte_flow_error *error)
16049 {
16050         struct mlx5_list_entry *entry;
16051         struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
16052         struct mlx5_flow_dv_matcher matcher = {
16053                 .mask = {
16054                         .size = sizeof(matcher.mask.buf),
16055                 },
16056                 .tbl = tbl_rsc,
16057         };
16058         struct mlx5_flow_dv_match_params value = {
16059                 .size = sizeof(value.buf),
16060         };
16061         struct mlx5_flow_cb_ctx ctx = {
16062                 .error = error,
16063                 .data = &matcher,
16064         };
16065         struct mlx5_flow_tbl_data_entry *tbl_data;
16066         struct mlx5_priv *priv = dev->data->dev_private;
16067         const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
16068
16069         if (match_src_port && (priv->representor || priv->master)) {
16070                 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
16071                                                    value.buf, item, attr)) {
16072                         DRV_LOG(ERR, "Failed to register meter policy%d matcher"
16073                                 " with port.", priority);
16074                         return -1;
16075                 }
16076         }
16077         tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
16078         if (priority < RTE_COLOR_RED)
16079                 flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16080                         (enum modify_reg)color_reg_c_idx, 0, color_mask);
16081         matcher.priority = priority;
16082         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
16083                                     matcher.mask.size);
16084         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16085         if (!entry) {
16086                 DRV_LOG(ERR, "Failed to register meter drop matcher.");
16087                 return -1;
16088         }
16089         *policy_matcher =
16090                 container_of(entry, struct mlx5_flow_dv_matcher, entry);
16091         return 0;
16092 }
16093
16094 /**
16095  * Create the policy rules per domain.
16096  *
16097  * @param[in] dev
16098  *   Pointer to Ethernet device.
16099  * @param[in] sub_policy
16100  *    Pointer to sub policy table..
16101  * @param[in] egress
16102  *   Direction of the table.
16103  * @param[in] transfer
16104  *   E-Switch or NIC flow.
16105  * @param[in] acts
16106  *   Pointer to policy action list per color.
16107  *
16108  * @return
16109  *   0 on success, -1 otherwise.
16110  */
16111 static int
16112 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
16113                 struct mlx5_flow_meter_sub_policy *sub_policy,
16114                 uint8_t egress, uint8_t transfer, bool match_src_port,
16115                 struct mlx5_meter_policy_acts acts[RTE_COLORS])
16116 {
16117         struct mlx5_priv *priv = dev->data->dev_private;
16118         struct rte_flow_error flow_err;
16119         uint32_t color_reg_c_idx;
16120         struct rte_flow_attr attr = {
16121                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16122                 .priority = 0,
16123                 .ingress = 0,
16124                 .egress = !!egress,
16125                 .transfer = !!transfer,
16126                 .reserved = 0,
16127         };
16128         int i;
16129         int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
16130         struct mlx5_sub_policy_color_rule *color_rule;
16131         bool svport_match;
16132         struct mlx5_sub_policy_color_rule *tmp_rules[RTE_COLORS] = {NULL};
16133
16134         if (ret < 0)
16135                 return -1;
16136         /* Create policy table with POLICY level. */
16137         if (!sub_policy->tbl_rsc)
16138                 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
16139                                 MLX5_FLOW_TABLE_LEVEL_POLICY,
16140                                 egress, transfer, false, NULL, 0, 0,
16141                                 sub_policy->idx, &flow_err);
16142         if (!sub_policy->tbl_rsc) {
16143                 DRV_LOG(ERR,
16144                         "Failed to create meter sub policy table.");
16145                 return -1;
16146         }
16147         /* Prepare matchers. */
16148         color_reg_c_idx = ret;
16149         for (i = 0; i < RTE_COLORS; i++) {
16150                 TAILQ_INIT(&sub_policy->color_rules[i]);
16151                 if (!acts[i].actions_n)
16152                         continue;
16153                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
16154                                 sizeof(struct mlx5_sub_policy_color_rule),
16155                                 0, SOCKET_ID_ANY);
16156                 if (!color_rule) {
16157                         DRV_LOG(ERR, "No memory to create color rule.");
16158                         goto err_exit;
16159                 }
16160                 tmp_rules[i] = color_rule;
16161                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
16162                                   color_rule, next_port);
16163                 color_rule->src_port = priv->representor_id;
16164                 /* No use. */
16165                 attr.priority = i;
16166                 /* Create matchers for colors. */
16167                 svport_match = (i != RTE_COLOR_RED) ? match_src_port : false;
16168                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
16169                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
16170                                 &attr, svport_match, NULL,
16171                                 &color_rule->matcher, &flow_err)) {
16172                         DRV_LOG(ERR, "Failed to create color%u matcher.", i);
16173                         goto err_exit;
16174                 }
16175                 /* Create flow, matching color. */
16176                 if (__flow_dv_create_policy_flow(dev,
16177                                 color_reg_c_idx, (enum rte_color)i,
16178                                 color_rule->matcher->matcher_object,
16179                                 acts[i].actions_n, acts[i].dv_actions,
16180                                 svport_match, NULL, &color_rule->rule,
16181                                 &attr)) {
16182                         DRV_LOG(ERR, "Failed to create color%u rule.", i);
16183                         goto err_exit;
16184                 }
16185         }
16186         return 0;
16187 err_exit:
16188         /* All the policy rules will be cleared. */
16189         do {
16190                 color_rule = tmp_rules[i];
16191                 if (color_rule) {
16192                         if (color_rule->rule)
16193                                 mlx5_flow_os_destroy_flow(color_rule->rule);
16194                         if (color_rule->matcher) {
16195                                 struct mlx5_flow_tbl_data_entry *tbl =
16196                                         container_of(color_rule->matcher->tbl,
16197                                                      typeof(*tbl), tbl);
16198                                 mlx5_list_unregister(tbl->matchers,
16199                                                 &color_rule->matcher->entry);
16200                         }
16201                         TAILQ_REMOVE(&sub_policy->color_rules[i],
16202                                      color_rule, next_port);
16203                         mlx5_free(color_rule);
16204                 }
16205         } while (i--);
16206         return -1;
16207 }
16208
16209 static int
16210 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
16211                         struct mlx5_flow_meter_policy *mtr_policy,
16212                         struct mlx5_flow_meter_sub_policy *sub_policy,
16213                         uint32_t domain)
16214 {
16215         struct mlx5_priv *priv = dev->data->dev_private;
16216         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16217         struct mlx5_flow_dv_tag_resource *tag;
16218         struct mlx5_flow_dv_port_id_action_resource *port_action;
16219         struct mlx5_hrxq *hrxq;
16220         struct mlx5_flow_meter_info *next_fm = NULL;
16221         struct mlx5_flow_meter_policy *next_policy;
16222         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16223         struct mlx5_flow_tbl_data_entry *tbl_data;
16224         struct rte_flow_error error;
16225         uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16226         uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16227         bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX);
16228         bool match_src_port = false;
16229         int i;
16230
16231         /* If RSS or Queue, no previous actions / rules is created. */
16232         for (i = 0; i < RTE_COLORS; i++) {
16233                 acts[i].actions_n = 0;
16234                 if (i == RTE_COLOR_RED) {
16235                         /* Only support drop on red. */
16236                         acts[i].dv_actions[0] =
16237                                 mtr_policy->dr_drop_action[domain];
16238                         acts[i].actions_n = 1;
16239                         continue;
16240                 }
16241                 if (i == RTE_COLOR_GREEN &&
16242                     mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
16243                         struct rte_flow_attr attr = {
16244                                 .transfer = transfer
16245                         };
16246
16247                         next_fm = mlx5_flow_meter_find(priv,
16248                                         mtr_policy->act_cnt[i].next_mtr_id,
16249                                         NULL);
16250                         if (!next_fm) {
16251                                 DRV_LOG(ERR,
16252                                         "Failed to get next hierarchy meter.");
16253                                 goto err_exit;
16254                         }
16255                         if (mlx5_flow_meter_attach(priv, next_fm,
16256                                                    &attr, &error)) {
16257                                 DRV_LOG(ERR, "%s", error.message);
16258                                 next_fm = NULL;
16259                                 goto err_exit;
16260                         }
16261                         /* Meter action must be the first for TX. */
16262                         if (mtr_first) {
16263                                 acts[i].dv_actions[acts[i].actions_n] =
16264                                         next_fm->meter_action;
16265                                 acts[i].actions_n++;
16266                         }
16267                 }
16268                 if (mtr_policy->act_cnt[i].rix_mark) {
16269                         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
16270                                         mtr_policy->act_cnt[i].rix_mark);
16271                         if (!tag) {
16272                                 DRV_LOG(ERR, "Failed to find "
16273                                 "mark action for policy.");
16274                                 goto err_exit;
16275                         }
16276                         acts[i].dv_actions[acts[i].actions_n] = tag->action;
16277                         acts[i].actions_n++;
16278                 }
16279                 if (mtr_policy->act_cnt[i].modify_hdr) {
16280                         acts[i].dv_actions[acts[i].actions_n] =
16281                                 mtr_policy->act_cnt[i].modify_hdr->action;
16282                         acts[i].actions_n++;
16283                 }
16284                 if (mtr_policy->act_cnt[i].fate_action) {
16285                         switch (mtr_policy->act_cnt[i].fate_action) {
16286                         case MLX5_FLOW_FATE_PORT_ID:
16287                                 port_action = mlx5_ipool_get
16288                                         (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
16289                                 mtr_policy->act_cnt[i].rix_port_id_action);
16290                                 if (!port_action) {
16291                                         DRV_LOG(ERR, "Failed to find "
16292                                                 "port action for policy.");
16293                                         goto err_exit;
16294                                 }
16295                                 acts[i].dv_actions[acts[i].actions_n] =
16296                                         port_action->action;
16297                                 acts[i].actions_n++;
16298                                 mtr_policy->dev = dev;
16299                                 match_src_port = true;
16300                                 break;
16301                         case MLX5_FLOW_FATE_DROP:
16302                         case MLX5_FLOW_FATE_JUMP:
16303                                 acts[i].dv_actions[acts[i].actions_n] =
16304                                 mtr_policy->act_cnt[i].dr_jump_action[domain];
16305                                 acts[i].actions_n++;
16306                                 break;
16307                         case MLX5_FLOW_FATE_SHARED_RSS:
16308                         case MLX5_FLOW_FATE_QUEUE:
16309                                 hrxq = mlx5_ipool_get
16310                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
16311                                          sub_policy->rix_hrxq[i]);
16312                                 if (!hrxq) {
16313                                         DRV_LOG(ERR, "Failed to find "
16314                                                 "queue action for policy.");
16315                                         goto err_exit;
16316                                 }
16317                                 acts[i].dv_actions[acts[i].actions_n] =
16318                                         hrxq->action;
16319                                 acts[i].actions_n++;
16320                                 break;
16321                         case MLX5_FLOW_FATE_MTR:
16322                                 if (!next_fm) {
16323                                         DRV_LOG(ERR,
16324                                                 "No next hierarchy meter.");
16325                                         goto err_exit;
16326                                 }
16327                                 if (!mtr_first) {
16328                                         acts[i].dv_actions[acts[i].actions_n] =
16329                                                         next_fm->meter_action;
16330                                         acts[i].actions_n++;
16331                                 }
16332                                 if (mtr_policy->act_cnt[i].next_sub_policy) {
16333                                         next_sub_policy =
16334                                         mtr_policy->act_cnt[i].next_sub_policy;
16335                                 } else {
16336                                         next_policy =
16337                                                 mlx5_flow_meter_policy_find(dev,
16338                                                 next_fm->policy_id, NULL);
16339                                         MLX5_ASSERT(next_policy);
16340                                         next_sub_policy =
16341                                         next_policy->sub_policys[domain][0];
16342                                 }
16343                                 tbl_data =
16344                                         container_of(next_sub_policy->tbl_rsc,
16345                                         struct mlx5_flow_tbl_data_entry, tbl);
16346                                 acts[i].dv_actions[acts[i].actions_n++] =
16347                                                         tbl_data->jump.action;
16348                                 if (mtr_policy->act_cnt[i].modify_hdr)
16349                                         match_src_port = !!transfer;
16350                                 break;
16351                         default:
16352                                 /*Queue action do nothing*/
16353                                 break;
16354                         }
16355                 }
16356         }
16357         if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
16358                                 egress, transfer, match_src_port, acts)) {
16359                 DRV_LOG(ERR,
16360                         "Failed to create policy rules per domain.");
16361                 goto err_exit;
16362         }
16363         return 0;
16364 err_exit:
16365         if (next_fm)
16366                 mlx5_flow_meter_detach(priv, next_fm);
16367         return -1;
16368 }
16369
16370 /**
16371  * Create the policy rules.
16372  *
16373  * @param[in] dev
16374  *   Pointer to Ethernet device.
16375  * @param[in,out] mtr_policy
16376  *   Pointer to meter policy table.
16377  *
16378  * @return
16379  *   0 on success, -1 otherwise.
16380  */
16381 static int
16382 flow_dv_create_policy_rules(struct rte_eth_dev *dev,
16383                              struct mlx5_flow_meter_policy *mtr_policy)
16384 {
16385         int i;
16386         uint16_t sub_policy_num;
16387
16388         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16389                 sub_policy_num = (mtr_policy->sub_policy_num >>
16390                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
16391                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16392                 if (!sub_policy_num)
16393                         continue;
16394                 /* Prepare actions list and create policy rules. */
16395                 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16396                         mtr_policy->sub_policys[i][0], i)) {
16397                         DRV_LOG(ERR, "Failed to create policy action "
16398                                 "list per domain.");
16399                         return -1;
16400                 }
16401         }
16402         return 0;
16403 }
16404
16405 static int
16406 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
16407 {
16408         struct mlx5_priv *priv = dev->data->dev_private;
16409         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16410         struct mlx5_flow_meter_def_policy *def_policy;
16411         struct mlx5_flow_tbl_resource *jump_tbl;
16412         struct mlx5_flow_tbl_data_entry *tbl_data;
16413         uint8_t egress, transfer;
16414         struct rte_flow_error error;
16415         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16416         int ret;
16417
16418         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16419         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16420         def_policy = mtrmng->def_policy[domain];
16421         if (!def_policy) {
16422                 def_policy = mlx5_malloc(MLX5_MEM_ZERO,
16423                         sizeof(struct mlx5_flow_meter_def_policy),
16424                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
16425                 if (!def_policy) {
16426                         DRV_LOG(ERR, "Failed to alloc default policy table.");
16427                         goto def_policy_error;
16428                 }
16429                 mtrmng->def_policy[domain] = def_policy;
16430                 /* Create the meter suffix table with SUFFIX level. */
16431                 jump_tbl = flow_dv_tbl_resource_get(dev,
16432                                 MLX5_FLOW_TABLE_LEVEL_METER,
16433                                 egress, transfer, false, NULL, 0,
16434                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16435                 if (!jump_tbl) {
16436                         DRV_LOG(ERR,
16437                                 "Failed to create meter suffix table.");
16438                         goto def_policy_error;
16439                 }
16440                 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
16441                 tbl_data = container_of(jump_tbl,
16442                                         struct mlx5_flow_tbl_data_entry, tbl);
16443                 def_policy->dr_jump_action[RTE_COLOR_GREEN] =
16444                                                 tbl_data->jump.action;
16445                 acts[RTE_COLOR_GREEN].dv_actions[0] = tbl_data->jump.action;
16446                 acts[RTE_COLOR_GREEN].actions_n = 1;
16447                 /*
16448                  * YELLOW has the same default policy as GREEN does.
16449                  * G & Y share the same table and action. The 2nd time of table
16450                  * resource getting is just to update the reference count for
16451                  * the releasing stage.
16452                  */
16453                 jump_tbl = flow_dv_tbl_resource_get(dev,
16454                                 MLX5_FLOW_TABLE_LEVEL_METER,
16455                                 egress, transfer, false, NULL, 0,
16456                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16457                 if (!jump_tbl) {
16458                         DRV_LOG(ERR,
16459                                 "Failed to get meter suffix table.");
16460                         goto def_policy_error;
16461                 }
16462                 def_policy->sub_policy.jump_tbl[RTE_COLOR_YELLOW] = jump_tbl;
16463                 tbl_data = container_of(jump_tbl,
16464                                         struct mlx5_flow_tbl_data_entry, tbl);
16465                 def_policy->dr_jump_action[RTE_COLOR_YELLOW] =
16466                                                 tbl_data->jump.action;
16467                 acts[RTE_COLOR_YELLOW].dv_actions[0] = tbl_data->jump.action;
16468                 acts[RTE_COLOR_YELLOW].actions_n = 1;
16469                 /* Create jump action to the drop table. */
16470                 if (!mtrmng->drop_tbl[domain]) {
16471                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
16472                                 (dev, MLX5_FLOW_TABLE_LEVEL_METER,
16473                                  egress, transfer, false, NULL, 0,
16474                                  0, MLX5_MTR_TABLE_ID_DROP, &error);
16475                         if (!mtrmng->drop_tbl[domain]) {
16476                                 DRV_LOG(ERR, "Failed to create meter "
16477                                         "drop table for default policy.");
16478                                 goto def_policy_error;
16479                         }
16480                 }
16481                 /* all RED: unique Drop table for jump action. */
16482                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16483                                         struct mlx5_flow_tbl_data_entry, tbl);
16484                 def_policy->dr_jump_action[RTE_COLOR_RED] =
16485                                                 tbl_data->jump.action;
16486                 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
16487                 acts[RTE_COLOR_RED].actions_n = 1;
16488                 /* Create default policy rules. */
16489                 ret = __flow_dv_create_domain_policy_rules(dev,
16490                                         &def_policy->sub_policy,
16491                                         egress, transfer, false, acts);
16492                 if (ret) {
16493                         DRV_LOG(ERR, "Failed to create default policy rules.");
16494                         goto def_policy_error;
16495                 }
16496         }
16497         return 0;
16498 def_policy_error:
16499         __flow_dv_destroy_domain_def_policy(dev,
16500                                             (enum mlx5_meter_domain)domain);
16501         return -1;
16502 }
16503
16504 /**
16505  * Create the default policy table set.
16506  *
16507  * @param[in] dev
16508  *   Pointer to Ethernet device.
16509  * @return
16510  *   0 on success, -1 otherwise.
16511  */
16512 static int
16513 flow_dv_create_def_policy(struct rte_eth_dev *dev)
16514 {
16515         struct mlx5_priv *priv = dev->data->dev_private;
16516         int i;
16517
16518         /* Non-termination policy table. */
16519         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16520                 if (!priv->config.dv_esw_en && i == MLX5_MTR_DOMAIN_TRANSFER)
16521                         continue;
16522                 if (__flow_dv_create_domain_def_policy(dev, i)) {
16523                         DRV_LOG(ERR, "Failed to create default policy");
16524                         /* Rollback the created default policies for others. */
16525                         flow_dv_destroy_def_policy(dev);
16526                         return -1;
16527                 }
16528         }
16529         return 0;
16530 }
16531
16532 /**
16533  * Create the needed meter tables.
16534  * Lock free, (mutex should be acquired by caller).
16535  *
16536  * @param[in] dev
16537  *   Pointer to Ethernet device.
16538  * @param[in] fm
16539  *   Meter information table.
16540  * @param[in] mtr_idx
16541  *   Meter index.
16542  * @param[in] domain_bitmap
16543  *   Domain bitmap.
16544  * @return
16545  *   0 on success, -1 otherwise.
16546  */
16547 static int
16548 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
16549                         struct mlx5_flow_meter_info *fm,
16550                         uint32_t mtr_idx,
16551                         uint8_t domain_bitmap)
16552 {
16553         struct mlx5_priv *priv = dev->data->dev_private;
16554         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16555         struct rte_flow_error error;
16556         struct mlx5_flow_tbl_data_entry *tbl_data;
16557         uint8_t egress, transfer;
16558         void *actions[METER_ACTIONS];
16559         int domain, ret, i;
16560         struct mlx5_flow_counter *cnt;
16561         struct mlx5_flow_dv_match_params value = {
16562                 .size = sizeof(value.buf),
16563         };
16564         struct mlx5_flow_dv_match_params matcher_para = {
16565                 .size = sizeof(matcher_para.buf),
16566         };
16567         int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
16568                                                      0, &error);
16569         uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
16570         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
16571         struct mlx5_list_entry *entry;
16572         struct mlx5_flow_dv_matcher matcher = {
16573                 .mask = {
16574                         .size = sizeof(matcher.mask.buf),
16575                 },
16576         };
16577         struct mlx5_flow_dv_matcher *drop_matcher;
16578         struct mlx5_flow_cb_ctx ctx = {
16579                 .error = &error,
16580                 .data = &matcher,
16581         };
16582         uint8_t misc_mask;
16583
16584         if (!priv->mtr_en || mtr_id_reg_c < 0) {
16585                 rte_errno = ENOTSUP;
16586                 return -1;
16587         }
16588         for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
16589                 if (!(domain_bitmap & (1 << domain)) ||
16590                         (mtrmng->def_rule[domain] && !fm->drop_cnt))
16591                         continue;
16592                 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16593                 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16594                 /* Create the drop table with METER DROP level. */
16595                 if (!mtrmng->drop_tbl[domain]) {
16596                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
16597                                         MLX5_FLOW_TABLE_LEVEL_METER,
16598                                         egress, transfer, false, NULL, 0,
16599                                         0, MLX5_MTR_TABLE_ID_DROP, &error);
16600                         if (!mtrmng->drop_tbl[domain]) {
16601                                 DRV_LOG(ERR, "Failed to create meter drop table.");
16602                                 goto policy_error;
16603                         }
16604                 }
16605                 /* Create default matcher in drop table. */
16606                 matcher.tbl = mtrmng->drop_tbl[domain],
16607                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16608                                 struct mlx5_flow_tbl_data_entry, tbl);
16609                 if (!mtrmng->def_matcher[domain]) {
16610                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16611                                        (enum modify_reg)mtr_id_reg_c,
16612                                        0, 0);
16613                         matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
16614                         matcher.crc = rte_raw_cksum
16615                                         ((const void *)matcher.mask.buf,
16616                                         matcher.mask.size);
16617                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16618                         if (!entry) {
16619                                 DRV_LOG(ERR, "Failed to register meter "
16620                                 "drop default matcher.");
16621                                 goto policy_error;
16622                         }
16623                         mtrmng->def_matcher[domain] = container_of(entry,
16624                         struct mlx5_flow_dv_matcher, entry);
16625                 }
16626                 /* Create default rule in drop table. */
16627                 if (!mtrmng->def_rule[domain]) {
16628                         i = 0;
16629                         actions[i++] = priv->sh->dr_drop_action;
16630                         flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16631                                 (enum modify_reg)mtr_id_reg_c, 0, 0);
16632                         misc_mask = flow_dv_matcher_enable(value.buf);
16633                         __flow_dv_adjust_buf_size(&value.size, misc_mask);
16634                         ret = mlx5_flow_os_create_flow
16635                                 (mtrmng->def_matcher[domain]->matcher_object,
16636                                 (void *)&value, i, actions,
16637                                 &mtrmng->def_rule[domain]);
16638                         if (ret) {
16639                                 DRV_LOG(ERR, "Failed to create meter "
16640                                 "default drop rule for drop table.");
16641                                 goto policy_error;
16642                         }
16643                 }
16644                 if (!fm->drop_cnt)
16645                         continue;
16646                 MLX5_ASSERT(mtrmng->max_mtr_bits);
16647                 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
16648                         /* Create matchers for Drop. */
16649                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16650                                         (enum modify_reg)mtr_id_reg_c, 0,
16651                                         (mtr_id_mask << mtr_id_offset));
16652                         matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
16653                         matcher.crc = rte_raw_cksum
16654                                         ((const void *)matcher.mask.buf,
16655                                         matcher.mask.size);
16656                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16657                         if (!entry) {
16658                                 DRV_LOG(ERR,
16659                                 "Failed to register meter drop matcher.");
16660                                 goto policy_error;
16661                         }
16662                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
16663                                 container_of(entry, struct mlx5_flow_dv_matcher,
16664                                              entry);
16665                 }
16666                 drop_matcher =
16667                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
16668                 /* Create drop rule, matching meter_id only. */
16669                 flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16670                                 (enum modify_reg)mtr_id_reg_c,
16671                                 (mtr_idx << mtr_id_offset), UINT32_MAX);
16672                 i = 0;
16673                 cnt = flow_dv_counter_get_by_idx(dev,
16674                                         fm->drop_cnt, NULL);
16675                 actions[i++] = cnt->action;
16676                 actions[i++] = priv->sh->dr_drop_action;
16677                 misc_mask = flow_dv_matcher_enable(value.buf);
16678                 __flow_dv_adjust_buf_size(&value.size, misc_mask);
16679                 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
16680                                                (void *)&value, i, actions,
16681                                                &fm->drop_rule[domain]);
16682                 if (ret) {
16683                         DRV_LOG(ERR, "Failed to create meter "
16684                                 "drop rule for drop table.");
16685                                 goto policy_error;
16686                 }
16687         }
16688         return 0;
16689 policy_error:
16690         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16691                 if (fm->drop_rule[i]) {
16692                         claim_zero(mlx5_flow_os_destroy_flow
16693                                 (fm->drop_rule[i]));
16694                         fm->drop_rule[i] = NULL;
16695                 }
16696         }
16697         return -1;
16698 }
16699
16700 static struct mlx5_flow_meter_sub_policy *
16701 __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
16702                 struct mlx5_flow_meter_policy *mtr_policy,
16703                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS],
16704                 struct mlx5_flow_meter_sub_policy *next_sub_policy,
16705                 bool *is_reuse)
16706 {
16707         struct mlx5_priv *priv = dev->data->dev_private;
16708         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16709         uint32_t sub_policy_idx = 0;
16710         uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
16711         uint32_t i, j;
16712         struct mlx5_hrxq *hrxq;
16713         struct mlx5_flow_handle dh;
16714         struct mlx5_meter_policy_action_container *act_cnt;
16715         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16716         uint16_t sub_policy_num;
16717
16718         rte_spinlock_lock(&mtr_policy->sl);
16719         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16720                 if (!rss_desc[i])
16721                         continue;
16722                 hrxq_idx[i] = mlx5_hrxq_get(dev, rss_desc[i]);
16723                 if (!hrxq_idx[i]) {
16724                         rte_spinlock_unlock(&mtr_policy->sl);
16725                         return NULL;
16726                 }
16727         }
16728         sub_policy_num = (mtr_policy->sub_policy_num >>
16729                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16730                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16731         for (j = 0; j < sub_policy_num; j++) {
16732                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16733                         if (rss_desc[i] &&
16734                             hrxq_idx[i] !=
16735                             mtr_policy->sub_policys[domain][j]->rix_hrxq[i])
16736                                 break;
16737                 }
16738                 if (i >= MLX5_MTR_RTE_COLORS) {
16739                         /*
16740                          * Found the sub policy table with
16741                          * the same queue per color.
16742                          */
16743                         rte_spinlock_unlock(&mtr_policy->sl);
16744                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
16745                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
16746                         *is_reuse = true;
16747                         return mtr_policy->sub_policys[domain][j];
16748                 }
16749         }
16750         /* Create sub policy. */
16751         if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
16752                 /* Reuse the first pre-allocated sub_policy. */
16753                 sub_policy = mtr_policy->sub_policys[domain][0];
16754                 sub_policy_idx = sub_policy->idx;
16755         } else {
16756                 sub_policy = mlx5_ipool_zmalloc
16757                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16758                                  &sub_policy_idx);
16759                 if (!sub_policy ||
16760                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
16761                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
16762                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
16763                         goto rss_sub_policy_error;
16764                 }
16765                 sub_policy->idx = sub_policy_idx;
16766                 sub_policy->main_policy = mtr_policy;
16767         }
16768         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16769                 if (!rss_desc[i])
16770                         continue;
16771                 sub_policy->rix_hrxq[i] = hrxq_idx[i];
16772                 if (mtr_policy->is_hierarchy) {
16773                         act_cnt = &mtr_policy->act_cnt[i];
16774                         act_cnt->next_sub_policy = next_sub_policy;
16775                         mlx5_hrxq_release(dev, hrxq_idx[i]);
16776                 } else {
16777                         /*
16778                          * Overwrite the last action from
16779                          * RSS action to Queue action.
16780                          */
16781                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
16782                                               hrxq_idx[i]);
16783                         if (!hrxq) {
16784                                 DRV_LOG(ERR, "Failed to get policy hrxq");
16785                                 goto rss_sub_policy_error;
16786                         }
16787                         act_cnt = &mtr_policy->act_cnt[i];
16788                         if (act_cnt->rix_mark || act_cnt->modify_hdr) {
16789                                 memset(&dh, 0, sizeof(struct mlx5_flow_handle));
16790                                 if (act_cnt->rix_mark)
16791                                         dh.mark = 1;
16792                                 dh.fate_action = MLX5_FLOW_FATE_QUEUE;
16793                                 dh.rix_hrxq = hrxq_idx[i];
16794                                 flow_drv_rxq_flags_set(dev, &dh);
16795                         }
16796                 }
16797         }
16798         if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16799                                                sub_policy, domain)) {
16800                 DRV_LOG(ERR, "Failed to create policy "
16801                         "rules for ingress domain.");
16802                 goto rss_sub_policy_error;
16803         }
16804         if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16805                 i = (mtr_policy->sub_policy_num >>
16806                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16807                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16808                 if (i >= MLX5_MTR_RSS_MAX_SUB_POLICY) {
16809                         DRV_LOG(ERR, "No free sub-policy slot.");
16810                         goto rss_sub_policy_error;
16811                 }
16812                 mtr_policy->sub_policys[domain][i] = sub_policy;
16813                 i++;
16814                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16815                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16816                 mtr_policy->sub_policy_num |=
16817                         (i & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16818                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16819         }
16820         rte_spinlock_unlock(&mtr_policy->sl);
16821         *is_reuse = false;
16822         return sub_policy;
16823 rss_sub_policy_error:
16824         if (sub_policy) {
16825                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16826                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16827                         i = (mtr_policy->sub_policy_num >>
16828                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16829                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16830                         mtr_policy->sub_policys[domain][i] = NULL;
16831                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16832                                         sub_policy->idx);
16833                 }
16834         }
16835         rte_spinlock_unlock(&mtr_policy->sl);
16836         return NULL;
16837 }
16838
16839 /**
16840  * Find the policy table for prefix table with RSS.
16841  *
16842  * @param[in] dev
16843  *   Pointer to Ethernet device.
16844  * @param[in] mtr_policy
16845  *   Pointer to meter policy table.
16846  * @param[in] rss_desc
16847  *   Pointer to rss_desc
16848  * @return
16849  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
16850  */
16851 static struct mlx5_flow_meter_sub_policy *
16852 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
16853                 struct mlx5_flow_meter_policy *mtr_policy,
16854                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
16855 {
16856         struct mlx5_priv *priv = dev->data->dev_private;
16857         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16858         struct mlx5_flow_meter_info *next_fm;
16859         struct mlx5_flow_meter_policy *next_policy;
16860         struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL;
16861         struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM];
16862         struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM];
16863         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16864         bool reuse_sub_policy;
16865         uint32_t i = 0;
16866         uint32_t j = 0;
16867
16868         while (true) {
16869                 /* Iterate hierarchy to get all policies in this hierarchy. */
16870                 policies[i++] = mtr_policy;
16871                 if (!mtr_policy->is_hierarchy)
16872                         break;
16873                 if (i >= MLX5_MTR_CHAIN_MAX_NUM) {
16874                         DRV_LOG(ERR, "Exceed max meter number in hierarchy.");
16875                         return NULL;
16876                 }
16877                 next_fm = mlx5_flow_meter_find(priv,
16878                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16879                 if (!next_fm) {
16880                         DRV_LOG(ERR, "Failed to get next meter in hierarchy.");
16881                         return NULL;
16882                 }
16883                 next_policy =
16884                         mlx5_flow_meter_policy_find(dev, next_fm->policy_id,
16885                                                     NULL);
16886                 MLX5_ASSERT(next_policy);
16887                 mtr_policy = next_policy;
16888         }
16889         while (i) {
16890                 /**
16891                  * From last policy to the first one in hierarchy,
16892                  * create / get the sub policy for each of them.
16893                  */
16894                 sub_policy = __flow_dv_meter_get_rss_sub_policy(dev,
16895                                                         policies[--i],
16896                                                         rss_desc,
16897                                                         next_sub_policy,
16898                                                         &reuse_sub_policy);
16899                 if (!sub_policy) {
16900                         DRV_LOG(ERR, "Failed to get the sub policy.");
16901                         goto err_exit;
16902                 }
16903                 if (!reuse_sub_policy)
16904                         sub_policies[j++] = sub_policy;
16905                 next_sub_policy = sub_policy;
16906         }
16907         return sub_policy;
16908 err_exit:
16909         while (j) {
16910                 uint16_t sub_policy_num;
16911
16912                 sub_policy = sub_policies[--j];
16913                 mtr_policy = sub_policy->main_policy;
16914                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16915                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16916                         sub_policy_num = (mtr_policy->sub_policy_num >>
16917                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16918                                 MLX5_MTR_SUB_POLICY_NUM_MASK;
16919                         mtr_policy->sub_policys[domain][sub_policy_num - 1] =
16920                                                                         NULL;
16921                         sub_policy_num--;
16922                         mtr_policy->sub_policy_num &=
16923                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16924                                   (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
16925                         mtr_policy->sub_policy_num |=
16926                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16927                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
16928                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16929                                         sub_policy->idx);
16930                 }
16931         }
16932         return NULL;
16933 }
16934
16935 /**
16936  * Create the sub policy tag rule for all meters in hierarchy.
16937  *
16938  * @param[in] dev
16939  *   Pointer to Ethernet device.
16940  * @param[in] fm
16941  *   Meter information table.
16942  * @param[in] src_port
16943  *   The src port this extra rule should use.
16944  * @param[in] item
16945  *   The src port match item.
16946  * @param[out] error
16947  *   Perform verbose error reporting if not NULL.
16948  * @return
16949  *   0 on success, a negative errno value otherwise and rte_errno is set.
16950  */
16951 static int
16952 flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
16953                                 struct mlx5_flow_meter_info *fm,
16954                                 int32_t src_port,
16955                                 const struct rte_flow_item *item,
16956                                 struct rte_flow_error *error)
16957 {
16958         struct mlx5_priv *priv = dev->data->dev_private;
16959         struct mlx5_flow_meter_policy *mtr_policy;
16960         struct mlx5_flow_meter_sub_policy *sub_policy;
16961         struct mlx5_flow_meter_info *next_fm = NULL;
16962         struct mlx5_flow_meter_policy *next_policy;
16963         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16964         struct mlx5_flow_tbl_data_entry *tbl_data;
16965         struct mlx5_sub_policy_color_rule *color_rule;
16966         struct mlx5_meter_policy_acts acts;
16967         uint32_t color_reg_c_idx;
16968         bool mtr_first = (src_port != UINT16_MAX) ? true : false;
16969         struct rte_flow_attr attr = {
16970                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16971                 .priority = 0,
16972                 .ingress = 0,
16973                 .egress = 0,
16974                 .transfer = 1,
16975                 .reserved = 0,
16976         };
16977         uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
16978         int i;
16979
16980         mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
16981         MLX5_ASSERT(mtr_policy);
16982         if (!mtr_policy->is_hierarchy)
16983                 return 0;
16984         next_fm = mlx5_flow_meter_find(priv,
16985                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16986         if (!next_fm) {
16987                 return rte_flow_error_set(error, EINVAL,
16988                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
16989                                 "Failed to find next meter in hierarchy.");
16990         }
16991         if (!next_fm->drop_cnt)
16992                 goto exit;
16993         color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error);
16994         sub_policy = mtr_policy->sub_policys[domain][0];
16995         for (i = 0; i < RTE_COLORS; i++) {
16996                 bool rule_exist = false;
16997                 struct mlx5_meter_policy_action_container *act_cnt;
16998
16999                 if (i >= RTE_COLOR_YELLOW)
17000                         break;
17001                 TAILQ_FOREACH(color_rule,
17002                               &sub_policy->color_rules[i], next_port)
17003                         if (color_rule->src_port == src_port) {
17004                                 rule_exist = true;
17005                                 break;
17006                         }
17007                 if (rule_exist)
17008                         continue;
17009                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
17010                                 sizeof(struct mlx5_sub_policy_color_rule),
17011                                 0, SOCKET_ID_ANY);
17012                 if (!color_rule)
17013                         return rte_flow_error_set(error, ENOMEM,
17014                                 RTE_FLOW_ERROR_TYPE_ACTION,
17015                                 NULL, "No memory to create tag color rule.");
17016                 color_rule->src_port = src_port;
17017                 attr.priority = i;
17018                 next_policy = mlx5_flow_meter_policy_find(dev,
17019                                                 next_fm->policy_id, NULL);
17020                 MLX5_ASSERT(next_policy);
17021                 next_sub_policy = next_policy->sub_policys[domain][0];
17022                 tbl_data = container_of(next_sub_policy->tbl_rsc,
17023                                         struct mlx5_flow_tbl_data_entry, tbl);
17024                 act_cnt = &mtr_policy->act_cnt[i];
17025                 if (mtr_first) {
17026                         acts.dv_actions[0] = next_fm->meter_action;
17027                         acts.dv_actions[1] = act_cnt->modify_hdr->action;
17028                 } else {
17029                         acts.dv_actions[0] = act_cnt->modify_hdr->action;
17030                         acts.dv_actions[1] = next_fm->meter_action;
17031                 }
17032                 acts.dv_actions[2] = tbl_data->jump.action;
17033                 acts.actions_n = 3;
17034                 if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) {
17035                         next_fm = NULL;
17036                         goto err_exit;
17037                 }
17038                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
17039                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
17040                                 &attr, true, item,
17041                                 &color_rule->matcher, error)) {
17042                         rte_flow_error_set(error, errno,
17043                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17044                                 "Failed to create hierarchy meter matcher.");
17045                         goto err_exit;
17046                 }
17047                 if (__flow_dv_create_policy_flow(dev, color_reg_c_idx,
17048                                         (enum rte_color)i,
17049                                         color_rule->matcher->matcher_object,
17050                                         acts.actions_n, acts.dv_actions,
17051                                         true, item,
17052                                         &color_rule->rule, &attr)) {
17053                         rte_flow_error_set(error, errno,
17054                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17055                                 "Failed to create hierarchy meter rule.");
17056                         goto err_exit;
17057                 }
17058                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
17059                                   color_rule, next_port);
17060         }
17061 exit:
17062         /**
17063          * Recursive call to iterate all meters in hierarchy and
17064          * create needed rules.
17065          */
17066         return flow_dv_meter_hierarchy_rule_create(dev, next_fm,
17067                                                 src_port, item, error);
17068 err_exit:
17069         if (color_rule) {
17070                 if (color_rule->rule)
17071                         mlx5_flow_os_destroy_flow(color_rule->rule);
17072                 if (color_rule->matcher) {
17073                         struct mlx5_flow_tbl_data_entry *tbl =
17074                                 container_of(color_rule->matcher->tbl,
17075                                                 typeof(*tbl), tbl);
17076                         mlx5_list_unregister(tbl->matchers,
17077                                                 &color_rule->matcher->entry);
17078                 }
17079                 mlx5_free(color_rule);
17080         }
17081         if (next_fm)
17082                 mlx5_flow_meter_detach(priv, next_fm);
17083         return -rte_errno;
17084 }
17085
17086 /**
17087  * Destroy the sub policy table with RX queue.
17088  *
17089  * @param[in] dev
17090  *   Pointer to Ethernet device.
17091  * @param[in] mtr_policy
17092  *   Pointer to meter policy table.
17093  */
17094 static void
17095 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
17096                                     struct mlx5_flow_meter_policy *mtr_policy)
17097 {
17098         struct mlx5_priv *priv = dev->data->dev_private;
17099         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
17100         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
17101         uint32_t i, j;
17102         uint16_t sub_policy_num, new_policy_num;
17103
17104         rte_spinlock_lock(&mtr_policy->sl);
17105         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17106                 switch (mtr_policy->act_cnt[i].fate_action) {
17107                 case MLX5_FLOW_FATE_SHARED_RSS:
17108                         sub_policy_num = (mtr_policy->sub_policy_num >>
17109                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17110                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17111                         new_policy_num = sub_policy_num;
17112                         for (j = 0; j < sub_policy_num; j++) {
17113                                 sub_policy =
17114                                         mtr_policy->sub_policys[domain][j];
17115                                 if (sub_policy) {
17116                                         __flow_dv_destroy_sub_policy_rules(dev,
17117                                                 sub_policy);
17118                                 if (sub_policy !=
17119                                         mtr_policy->sub_policys[domain][0]) {
17120                                         mtr_policy->sub_policys[domain][j] =
17121                                                                 NULL;
17122                                         mlx5_ipool_free
17123                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17124                                                 sub_policy->idx);
17125                                                 new_policy_num--;
17126                                         }
17127                                 }
17128                         }
17129                         if (new_policy_num != sub_policy_num) {
17130                                 mtr_policy->sub_policy_num &=
17131                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
17132                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
17133                                 mtr_policy->sub_policy_num |=
17134                                 (new_policy_num &
17135                                         MLX5_MTR_SUB_POLICY_NUM_MASK) <<
17136                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
17137                         }
17138                         break;
17139                 case MLX5_FLOW_FATE_QUEUE:
17140                         sub_policy = mtr_policy->sub_policys[domain][0];
17141                         __flow_dv_destroy_sub_policy_rules(dev,
17142                                                            sub_policy);
17143                         break;
17144                 default:
17145                         /*Other actions without queue and do nothing*/
17146                         break;
17147                 }
17148         }
17149         rte_spinlock_unlock(&mtr_policy->sl);
17150 }
17151 /**
17152  * Check whether the DR drop action is supported on the root table or not.
17153  *
17154  * Create a simple flow with DR drop action on root table to validate
17155  * if DR drop action on root table is supported or not.
17156  *
17157  * @param[in] dev
17158  *   Pointer to rte_eth_dev structure.
17159  *
17160  * @return
17161  *   0 on success, a negative errno value otherwise and rte_errno is set.
17162  */
17163 int
17164 mlx5_flow_discover_dr_action_support(struct rte_eth_dev *dev)
17165 {
17166         struct mlx5_priv *priv = dev->data->dev_private;
17167         struct mlx5_dev_ctx_shared *sh = priv->sh;
17168         struct mlx5_flow_dv_match_params mask = {
17169                 .size = sizeof(mask.buf),
17170         };
17171         struct mlx5_flow_dv_match_params value = {
17172                 .size = sizeof(value.buf),
17173         };
17174         struct mlx5dv_flow_matcher_attr dv_attr = {
17175                 .type = IBV_FLOW_ATTR_NORMAL,
17176                 .priority = 0,
17177                 .match_criteria_enable = 0,
17178                 .match_mask = (void *)&mask,
17179         };
17180         struct mlx5_flow_tbl_resource *tbl = NULL;
17181         void *matcher = NULL;
17182         void *flow = NULL;
17183         int ret = -1;
17184
17185         tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL,
17186                                         0, 0, 0, NULL);
17187         if (!tbl)
17188                 goto err;
17189         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
17190         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
17191         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
17192                                                &matcher);
17193         if (ret)
17194                 goto err;
17195         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
17196         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
17197                                        &sh->dr_drop_action, &flow);
17198 err:
17199         /*
17200          * If DR drop action is not supported on root table, flow create will
17201          * be failed with EOPNOTSUPP or EPROTONOSUPPORT.
17202          */
17203         if (!flow) {
17204                 if (matcher &&
17205                     (errno == EPROTONOSUPPORT || errno == EOPNOTSUPP))
17206                         DRV_LOG(INFO, "DR drop action is not supported in root table.");
17207                 else
17208                         DRV_LOG(ERR, "Unexpected error in DR drop action support detection");
17209                 ret = -1;
17210         } else {
17211                 claim_zero(mlx5_flow_os_destroy_flow(flow));
17212         }
17213         if (matcher)
17214                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
17215         if (tbl)
17216                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
17217         return ret;
17218 }
17219
17220 /**
17221  * Validate the batch counter support in root table.
17222  *
17223  * Create a simple flow with invalid counter and drop action on root table to
17224  * validate if batch counter with offset on root table is supported or not.
17225  *
17226  * @param[in] dev
17227  *   Pointer to rte_eth_dev structure.
17228  *
17229  * @return
17230  *   0 on success, a negative errno value otherwise and rte_errno is set.
17231  */
17232 int
17233 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
17234 {
17235         struct mlx5_priv *priv = dev->data->dev_private;
17236         struct mlx5_dev_ctx_shared *sh = priv->sh;
17237         struct mlx5_flow_dv_match_params mask = {
17238                 .size = sizeof(mask.buf),
17239         };
17240         struct mlx5_flow_dv_match_params value = {
17241                 .size = sizeof(value.buf),
17242         };
17243         struct mlx5dv_flow_matcher_attr dv_attr = {
17244                 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS,
17245                 .priority = 0,
17246                 .match_criteria_enable = 0,
17247                 .match_mask = (void *)&mask,
17248         };
17249         void *actions[2] = { 0 };
17250         struct mlx5_flow_tbl_resource *tbl = NULL;
17251         struct mlx5_devx_obj *dcs = NULL;
17252         void *matcher = NULL;
17253         void *flow = NULL;
17254         int ret = -1;
17255
17256         tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL,
17257                                         0, 0, 0, NULL);
17258         if (!tbl)
17259                 goto err;
17260         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
17261         if (!dcs)
17262                 goto err;
17263         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
17264                                                     &actions[0]);
17265         if (ret)
17266                 goto err;
17267         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
17268         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
17269         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
17270                                                &matcher);
17271         if (ret)
17272                 goto err;
17273         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
17274         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
17275                                        actions, &flow);
17276 err:
17277         /*
17278          * If batch counter with offset is not supported, the driver will not
17279          * validate the invalid offset value, flow create should success.
17280          * In this case, it means batch counter is not supported in root table.
17281          *
17282          * Otherwise, if flow create is failed, counter offset is supported.
17283          */
17284         if (flow) {
17285                 DRV_LOG(INFO, "Batch counter is not supported in root "
17286                               "table. Switch to fallback mode.");
17287                 rte_errno = ENOTSUP;
17288                 ret = -rte_errno;
17289                 claim_zero(mlx5_flow_os_destroy_flow(flow));
17290         } else {
17291                 /* Check matcher to make sure validate fail at flow create. */
17292                 if (!matcher || (matcher && errno != EINVAL))
17293                         DRV_LOG(ERR, "Unexpected error in counter offset "
17294                                      "support detection");
17295                 ret = 0;
17296         }
17297         if (actions[0])
17298                 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0]));
17299         if (matcher)
17300                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
17301         if (tbl)
17302                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
17303         if (dcs)
17304                 claim_zero(mlx5_devx_cmd_destroy(dcs));
17305         return ret;
17306 }
17307
17308 /**
17309  * Query a devx counter.
17310  *
17311  * @param[in] dev
17312  *   Pointer to the Ethernet device structure.
17313  * @param[in] cnt
17314  *   Index to the flow counter.
17315  * @param[in] clear
17316  *   Set to clear the counter statistics.
17317  * @param[out] pkts
17318  *   The statistics value of packets.
17319  * @param[out] bytes
17320  *   The statistics value of bytes.
17321  *
17322  * @return
17323  *   0 on success, otherwise return -1.
17324  */
17325 static int
17326 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
17327                       uint64_t *pkts, uint64_t *bytes)
17328 {
17329         struct mlx5_priv *priv = dev->data->dev_private;
17330         struct mlx5_flow_counter *cnt;
17331         uint64_t inn_pkts, inn_bytes;
17332         int ret;
17333
17334         if (!priv->config.devx)
17335                 return -1;
17336
17337         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
17338         if (ret)
17339                 return -1;
17340         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
17341         *pkts = inn_pkts - cnt->hits;
17342         *bytes = inn_bytes - cnt->bytes;
17343         if (clear) {
17344                 cnt->hits = inn_pkts;
17345                 cnt->bytes = inn_bytes;
17346         }
17347         return 0;
17348 }
17349
17350 /**
17351  * Get aged-out flows.
17352  *
17353  * @param[in] dev
17354  *   Pointer to the Ethernet device structure.
17355  * @param[in] context
17356  *   The address of an array of pointers to the aged-out flows contexts.
17357  * @param[in] nb_contexts
17358  *   The length of context array pointers.
17359  * @param[out] error
17360  *   Perform verbose error reporting if not NULL. Initialized in case of
17361  *   error only.
17362  *
17363  * @return
17364  *   how many contexts get in success, otherwise negative errno value.
17365  *   if nb_contexts is 0, return the amount of all aged contexts.
17366  *   if nb_contexts is not 0 , return the amount of aged flows reported
17367  *   in the context array.
17368  * @note: only stub for now
17369  */
17370 static int
17371 flow_dv_get_aged_flows(struct rte_eth_dev *dev,
17372                     void **context,
17373                     uint32_t nb_contexts,
17374                     struct rte_flow_error *error)
17375 {
17376         struct mlx5_priv *priv = dev->data->dev_private;
17377         struct mlx5_age_info *age_info;
17378         struct mlx5_age_param *age_param;
17379         struct mlx5_flow_counter *counter;
17380         struct mlx5_aso_age_action *act;
17381         int nb_flows = 0;
17382
17383         if (nb_contexts && !context)
17384                 return rte_flow_error_set(error, EINVAL,
17385                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17386                                           NULL, "empty context");
17387         age_info = GET_PORT_AGE_INFO(priv);
17388         rte_spinlock_lock(&age_info->aged_sl);
17389         LIST_FOREACH(act, &age_info->aged_aso, next) {
17390                 nb_flows++;
17391                 if (nb_contexts) {
17392                         context[nb_flows - 1] =
17393                                                 act->age_params.context;
17394                         if (!(--nb_contexts))
17395                                 break;
17396                 }
17397         }
17398         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
17399                 nb_flows++;
17400                 if (nb_contexts) {
17401                         age_param = MLX5_CNT_TO_AGE(counter);
17402                         context[nb_flows - 1] = age_param->context;
17403                         if (!(--nb_contexts))
17404                                 break;
17405                 }
17406         }
17407         rte_spinlock_unlock(&age_info->aged_sl);
17408         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
17409         return nb_flows;
17410 }
17411
17412 /*
17413  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
17414  */
17415 static uint32_t
17416 flow_dv_counter_allocate(struct rte_eth_dev *dev)
17417 {
17418         return flow_dv_counter_alloc(dev, 0);
17419 }
17420
17421 /**
17422  * Validate indirect action.
17423  * Dispatcher for action type specific validation.
17424  *
17425  * @param[in] dev
17426  *   Pointer to the Ethernet device structure.
17427  * @param[in] conf
17428  *   Indirect action configuration.
17429  * @param[in] action
17430  *   The indirect action object to validate.
17431  * @param[out] error
17432  *   Perform verbose error reporting if not NULL. Initialized in case of
17433  *   error only.
17434  *
17435  * @return
17436  *   0 on success, otherwise negative errno value.
17437  */
17438 static int
17439 flow_dv_action_validate(struct rte_eth_dev *dev,
17440                         const struct rte_flow_indir_action_conf *conf,
17441                         const struct rte_flow_action *action,
17442                         struct rte_flow_error *err)
17443 {
17444         struct mlx5_priv *priv = dev->data->dev_private;
17445
17446         RTE_SET_USED(conf);
17447         switch (action->type) {
17448         case RTE_FLOW_ACTION_TYPE_RSS:
17449                 /*
17450                  * priv->obj_ops is set according to driver capabilities.
17451                  * When DevX capabilities are
17452                  * sufficient, it is set to devx_obj_ops.
17453                  * Otherwise, it is set to ibv_obj_ops.
17454                  * ibv_obj_ops doesn't support ind_table_modify operation.
17455                  * In this case the indirect RSS action can't be used.
17456                  */
17457                 if (priv->obj_ops.ind_table_modify == NULL)
17458                         return rte_flow_error_set
17459                                         (err, ENOTSUP,
17460                                          RTE_FLOW_ERROR_TYPE_ACTION,
17461                                          NULL,
17462                                          "Indirect RSS action not supported");
17463                 return mlx5_validate_action_rss(dev, action, err);
17464         case RTE_FLOW_ACTION_TYPE_AGE:
17465                 if (!priv->sh->aso_age_mng)
17466                         return rte_flow_error_set(err, ENOTSUP,
17467                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17468                                                 NULL,
17469                                                 "Indirect age action not supported");
17470                 return flow_dv_validate_action_age(0, action, dev, err);
17471         case RTE_FLOW_ACTION_TYPE_COUNT:
17472                 /*
17473                  * There are two mechanisms to share the action count.
17474                  * The old mechanism uses the shared field to share, while the
17475                  * new mechanism uses the indirect action API.
17476                  * This validation comes to make sure that the two mechanisms
17477                  * are not combined.
17478                  */
17479                 if (is_shared_action_count(action))
17480                         return rte_flow_error_set(err, ENOTSUP,
17481                                                   RTE_FLOW_ERROR_TYPE_ACTION,
17482                                                   NULL,
17483                                                   "Mix shared and indirect counter is not supported");
17484                 return flow_dv_validate_action_count(dev, true, 0, err);
17485         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
17486                 if (!priv->sh->ct_aso_en)
17487                         return rte_flow_error_set(err, ENOTSUP,
17488                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17489                                         "ASO CT is not supported");
17490                 return mlx5_validate_action_ct(dev, action->conf, err);
17491         default:
17492                 return rte_flow_error_set(err, ENOTSUP,
17493                                           RTE_FLOW_ERROR_TYPE_ACTION,
17494                                           NULL,
17495                                           "action type not supported");
17496         }
17497 }
17498
17499 /*
17500  * Check if the RSS configurations for colors of a meter policy match
17501  * each other, except the queues.
17502  *
17503  * @param[in] r1
17504  *   Pointer to the first RSS flow action.
17505  * @param[in] r2
17506  *   Pointer to the second RSS flow action.
17507  *
17508  * @return
17509  *   0 on match, 1 on conflict.
17510  */
17511 static inline int
17512 flow_dv_mtr_policy_rss_compare(const struct rte_flow_action_rss *r1,
17513                                const struct rte_flow_action_rss *r2)
17514 {
17515         if (!r1 || !r2)
17516                 return 0;
17517         if (r1->func != r2->func || r1->level != r2->level ||
17518             r1->types != r2->types || r1->key_len != r2->key_len ||
17519             memcmp(r1->key, r2->key, r1->key_len))
17520                 return 1;
17521         return 0;
17522 }
17523
17524 /**
17525  * Validate the meter hierarchy chain for meter policy.
17526  *
17527  * @param[in] dev
17528  *   Pointer to the Ethernet device structure.
17529  * @param[in] meter_id
17530  *   Meter id.
17531  * @param[in] action_flags
17532  *   Holds the actions detected until now.
17533  * @param[out] is_rss
17534  *   Is RSS or not.
17535  * @param[out] hierarchy_domain
17536  *   The domain bitmap for hierarchy policy.
17537  * @param[out] error
17538  *   Perform verbose error reporting if not NULL. Initialized in case of
17539  *   error only.
17540  *
17541  * @return
17542  *   0 on success, otherwise negative errno value with error set.
17543  */
17544 static int
17545 flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
17546                                   uint32_t meter_id,
17547                                   uint64_t action_flags,
17548                                   bool *is_rss,
17549                                   uint8_t *hierarchy_domain,
17550                                   struct rte_mtr_error *error)
17551 {
17552         struct mlx5_priv *priv = dev->data->dev_private;
17553         struct mlx5_flow_meter_info *fm;
17554         struct mlx5_flow_meter_policy *policy;
17555         uint8_t cnt = 1;
17556
17557         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
17558                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17559                 return -rte_mtr_error_set(error, EINVAL,
17560                                         RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN,
17561                                         NULL,
17562                                         "Multiple fate actions not supported.");
17563         *hierarchy_domain = 0;
17564         while (true) {
17565                 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
17566                 if (!fm)
17567                         return -rte_mtr_error_set(error, EINVAL,
17568                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17569                                         "Meter not found in meter hierarchy.");
17570                 if (fm->def_policy)
17571                         return -rte_mtr_error_set(error, EINVAL,
17572                                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17573                         "Non termination meter not supported in hierarchy.");
17574                 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
17575                 MLX5_ASSERT(policy);
17576                 /**
17577                  * Only inherit the supported domains of the first meter in
17578                  * hierarchy.
17579                  * One meter supports at least one domain.
17580                  */
17581                 if (!*hierarchy_domain) {
17582                         if (policy->transfer)
17583                                 *hierarchy_domain |=
17584                                                 MLX5_MTR_DOMAIN_TRANSFER_BIT;
17585                         if (policy->ingress)
17586                                 *hierarchy_domain |=
17587                                                 MLX5_MTR_DOMAIN_INGRESS_BIT;
17588                         if (policy->egress)
17589                                 *hierarchy_domain |= MLX5_MTR_DOMAIN_EGRESS_BIT;
17590                 }
17591                 if (!policy->is_hierarchy) {
17592                         *is_rss = policy->is_rss;
17593                         break;
17594                 }
17595                 meter_id = policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id;
17596                 if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM)
17597                         return -rte_mtr_error_set(error, EINVAL,
17598                                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
17599                                         "Exceed max hierarchy meter number.");
17600         }
17601         return 0;
17602 }
17603
17604 /**
17605  * Validate meter policy actions.
17606  * Dispatcher for action type specific validation.
17607  *
17608  * @param[in] dev
17609  *   Pointer to the Ethernet device structure.
17610  * @param[in] action
17611  *   The meter policy action object to validate.
17612  * @param[in] attr
17613  *   Attributes of flow to determine steering domain.
17614  * @param[out] error
17615  *   Perform verbose error reporting if not NULL. Initialized in case of
17616  *   error only.
17617  *
17618  * @return
17619  *   0 on success, otherwise negative errno value.
17620  */
17621 static int
17622 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
17623                         const struct rte_flow_action *actions[RTE_COLORS],
17624                         struct rte_flow_attr *attr,
17625                         bool *is_rss,
17626                         uint8_t *domain_bitmap,
17627                         uint8_t *policy_mode,
17628                         struct rte_mtr_error *error)
17629 {
17630         struct mlx5_priv *priv = dev->data->dev_private;
17631         struct mlx5_dev_config *dev_conf = &priv->config;
17632         const struct rte_flow_action *act;
17633         uint64_t action_flags[RTE_COLORS] = {0};
17634         int actions_n;
17635         int i, ret;
17636         struct rte_flow_error flow_err;
17637         uint8_t domain_color[RTE_COLORS] = {0};
17638         uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
17639         uint8_t hierarchy_domain = 0;
17640         const struct rte_flow_action_meter *mtr;
17641         bool def_green = false;
17642         bool def_yellow = false;
17643         const struct rte_flow_action_rss *rss_color[RTE_COLORS] = {NULL};
17644
17645         if (!priv->config.dv_esw_en)
17646                 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
17647         *domain_bitmap = def_domain;
17648         /* Red color could only support DROP action. */
17649         if (!actions[RTE_COLOR_RED] ||
17650             actions[RTE_COLOR_RED]->type != RTE_FLOW_ACTION_TYPE_DROP)
17651                 return -rte_mtr_error_set(error, ENOTSUP,
17652                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17653                                 NULL, "Red color only supports drop action.");
17654         /*
17655          * Check default policy actions:
17656          * Green / Yellow: no action, Red: drop action
17657          * Either G or Y will trigger default policy actions to be created.
17658          */
17659         if (!actions[RTE_COLOR_GREEN] ||
17660             actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END)
17661                 def_green = true;
17662         if (!actions[RTE_COLOR_YELLOW] ||
17663             actions[RTE_COLOR_YELLOW]->type == RTE_FLOW_ACTION_TYPE_END)
17664                 def_yellow = true;
17665         if (def_green && def_yellow) {
17666                 *policy_mode = MLX5_MTR_POLICY_MODE_DEF;
17667                 return 0;
17668         } else if (!def_green && def_yellow) {
17669                 *policy_mode = MLX5_MTR_POLICY_MODE_OG;
17670         } else if (def_green && !def_yellow) {
17671                 *policy_mode = MLX5_MTR_POLICY_MODE_OY;
17672         }
17673         /* Set to empty string in case of NULL pointer access by user. */
17674         flow_err.message = "";
17675         for (i = 0; i < RTE_COLORS; i++) {
17676                 act = actions[i];
17677                 for (action_flags[i] = 0, actions_n = 0;
17678                      act && act->type != RTE_FLOW_ACTION_TYPE_END;
17679                      act++) {
17680                         if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
17681                                 return -rte_mtr_error_set(error, ENOTSUP,
17682                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17683                                           NULL, "too many actions");
17684                         switch (act->type) {
17685                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
17686                                 if (!priv->config.dv_esw_en)
17687                                         return -rte_mtr_error_set(error,
17688                                         ENOTSUP,
17689                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17690                                         NULL, "PORT action validate check"
17691                                         " fail for ESW disable");
17692                                 ret = flow_dv_validate_action_port_id(dev,
17693                                                 action_flags[i],
17694                                                 act, attr, &flow_err);
17695                                 if (ret)
17696                                         return -rte_mtr_error_set(error,
17697                                         ENOTSUP,
17698                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17699                                         NULL, flow_err.message ?
17700                                         flow_err.message :
17701                                         "PORT action validate check fail");
17702                                 ++actions_n;
17703                                 action_flags[i] |= MLX5_FLOW_ACTION_PORT_ID;
17704                                 break;
17705                         case RTE_FLOW_ACTION_TYPE_MARK:
17706                                 ret = flow_dv_validate_action_mark(dev, act,
17707                                                            action_flags[i],
17708                                                            attr, &flow_err);
17709                                 if (ret < 0)
17710                                         return -rte_mtr_error_set(error,
17711                                         ENOTSUP,
17712                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17713                                         NULL, flow_err.message ?
17714                                         flow_err.message :
17715                                         "Mark action validate check fail");
17716                                 if (dev_conf->dv_xmeta_en !=
17717                                         MLX5_XMETA_MODE_LEGACY)
17718                                         return -rte_mtr_error_set(error,
17719                                         ENOTSUP,
17720                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17721                                         NULL, "Extend MARK action is "
17722                                         "not supported. Please try use "
17723                                         "default policy for meter.");
17724                                 action_flags[i] |= MLX5_FLOW_ACTION_MARK;
17725                                 ++actions_n;
17726                                 break;
17727                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
17728                                 ret = flow_dv_validate_action_set_tag(dev,
17729                                                         act, action_flags[i],
17730                                                         attr, &flow_err);
17731                                 if (ret)
17732                                         return -rte_mtr_error_set(error,
17733                                         ENOTSUP,
17734                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17735                                         NULL, flow_err.message ?
17736                                         flow_err.message :
17737                                         "Set tag action validate check fail");
17738                                 action_flags[i] |= MLX5_FLOW_ACTION_SET_TAG;
17739                                 ++actions_n;
17740                                 break;
17741                         case RTE_FLOW_ACTION_TYPE_DROP:
17742                                 ret = mlx5_flow_validate_action_drop
17743                                         (action_flags[i], attr, &flow_err);
17744                                 if (ret < 0)
17745                                         return -rte_mtr_error_set(error,
17746                                         ENOTSUP,
17747                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17748                                         NULL, flow_err.message ?
17749                                         flow_err.message :
17750                                         "Drop action validate check fail");
17751                                 action_flags[i] |= MLX5_FLOW_ACTION_DROP;
17752                                 ++actions_n;
17753                                 break;
17754                         case RTE_FLOW_ACTION_TYPE_QUEUE:
17755                                 /*
17756                                  * Check whether extensive
17757                                  * metadata feature is engaged.
17758                                  */
17759                                 if (dev_conf->dv_flow_en &&
17760                                     (dev_conf->dv_xmeta_en !=
17761                                      MLX5_XMETA_MODE_LEGACY) &&
17762                                     mlx5_flow_ext_mreg_supported(dev))
17763                                         return -rte_mtr_error_set(error,
17764                                           ENOTSUP,
17765                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17766                                           NULL, "Queue action with meta "
17767                                           "is not supported. Please try use "
17768                                           "default policy for meter.");
17769                                 ret = mlx5_flow_validate_action_queue(act,
17770                                                         action_flags[i], dev,
17771                                                         attr, &flow_err);
17772                                 if (ret < 0)
17773                                         return -rte_mtr_error_set(error,
17774                                           ENOTSUP,
17775                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17776                                           NULL, flow_err.message ?
17777                                           flow_err.message :
17778                                           "Queue action validate check fail");
17779                                 action_flags[i] |= MLX5_FLOW_ACTION_QUEUE;
17780                                 ++actions_n;
17781                                 break;
17782                         case RTE_FLOW_ACTION_TYPE_RSS:
17783                                 if (dev_conf->dv_flow_en &&
17784                                     (dev_conf->dv_xmeta_en !=
17785                                      MLX5_XMETA_MODE_LEGACY) &&
17786                                     mlx5_flow_ext_mreg_supported(dev))
17787                                         return -rte_mtr_error_set(error,
17788                                           ENOTSUP,
17789                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17790                                           NULL, "RSS action with meta "
17791                                           "is not supported. Please try use "
17792                                           "default policy for meter.");
17793                                 ret = mlx5_validate_action_rss(dev, act,
17794                                                                &flow_err);
17795                                 if (ret < 0)
17796                                         return -rte_mtr_error_set(error,
17797                                           ENOTSUP,
17798                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17799                                           NULL, flow_err.message ?
17800                                           flow_err.message :
17801                                           "RSS action validate check fail");
17802                                 action_flags[i] |= MLX5_FLOW_ACTION_RSS;
17803                                 ++actions_n;
17804                                 /* Either G or Y will set the RSS. */
17805                                 rss_color[i] = act->conf;
17806                                 break;
17807                         case RTE_FLOW_ACTION_TYPE_JUMP:
17808                                 ret = flow_dv_validate_action_jump(dev,
17809                                         NULL, act, action_flags[i],
17810                                         attr, true, &flow_err);
17811                                 if (ret)
17812                                         return -rte_mtr_error_set(error,
17813                                           ENOTSUP,
17814                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17815                                           NULL, flow_err.message ?
17816                                           flow_err.message :
17817                                           "Jump action validate check fail");
17818                                 ++actions_n;
17819                                 action_flags[i] |= MLX5_FLOW_ACTION_JUMP;
17820                                 break;
17821                         /*
17822                          * Only the last meter in the hierarchy will support
17823                          * the YELLOW color steering. Then in the meter policy
17824                          * actions list, there should be no other meter inside.
17825                          */
17826                         case RTE_FLOW_ACTION_TYPE_METER:
17827                                 if (i != RTE_COLOR_GREEN)
17828                                         return -rte_mtr_error_set(error,
17829                                                 ENOTSUP,
17830                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17831                                                 NULL,
17832                                                 "Meter hierarchy only supports GREEN color.");
17833                                 if (*policy_mode != MLX5_MTR_POLICY_MODE_OG)
17834                                         return -rte_mtr_error_set(error,
17835                                                 ENOTSUP,
17836                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17837                                                 NULL,
17838                                                 "No yellow policy should be provided in meter hierarchy.");
17839                                 mtr = act->conf;
17840                                 ret = flow_dv_validate_policy_mtr_hierarchy(dev,
17841                                                         mtr->mtr_id,
17842                                                         action_flags[i],
17843                                                         is_rss,
17844                                                         &hierarchy_domain,
17845                                                         error);
17846                                 if (ret)
17847                                         return ret;
17848                                 ++actions_n;
17849                                 action_flags[i] |=
17850                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
17851                                 break;
17852                         default:
17853                                 return -rte_mtr_error_set(error, ENOTSUP,
17854                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17855                                         NULL,
17856                                         "Doesn't support optional action");
17857                         }
17858                 }
17859                 if (action_flags[i] & MLX5_FLOW_ACTION_PORT_ID)
17860                         domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT;
17861                 else if ((action_flags[i] &
17862                           (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) ||
17863                          (action_flags[i] & MLX5_FLOW_ACTION_MARK))
17864                         /*
17865                          * Only support MLX5_XMETA_MODE_LEGACY
17866                          * so MARK action is only in ingress domain.
17867                          */
17868                         domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT;
17869                 else
17870                         domain_color[i] = def_domain;
17871                 if (action_flags[i] &
17872                     MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
17873                         domain_color[i] &= hierarchy_domain;
17874                 /*
17875                  * Non-termination actions only support NIC Tx domain.
17876                  * The adjustion should be skipped when there is no
17877                  * action or only END is provided. The default domains
17878                  * bit-mask is set to find the MIN intersection.
17879                  * The action flags checking should also be skipped.
17880                  */
17881                 if ((def_green && i == RTE_COLOR_GREEN) ||
17882                     (def_yellow && i == RTE_COLOR_YELLOW))
17883                         continue;
17884                 /*
17885                  * Validate the drop action mutual exclusion
17886                  * with other actions. Drop action is mutually-exclusive
17887                  * with any other action, except for Count action.
17888                  */
17889                 if ((action_flags[i] & MLX5_FLOW_ACTION_DROP) &&
17890                     (action_flags[i] & ~MLX5_FLOW_ACTION_DROP)) {
17891                         return -rte_mtr_error_set(error, ENOTSUP,
17892                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17893                                 NULL, "Drop action is mutually-exclusive "
17894                                 "with any other action");
17895                 }
17896                 /* Eswitch has few restrictions on using items and actions */
17897                 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) {
17898                         if (!mlx5_flow_ext_mreg_supported(dev) &&
17899                             action_flags[i] & MLX5_FLOW_ACTION_MARK)
17900                                 return -rte_mtr_error_set(error, ENOTSUP,
17901                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17902                                         NULL, "unsupported action MARK");
17903                         if (action_flags[i] & MLX5_FLOW_ACTION_QUEUE)
17904                                 return -rte_mtr_error_set(error, ENOTSUP,
17905                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17906                                         NULL, "unsupported action QUEUE");
17907                         if (action_flags[i] & MLX5_FLOW_ACTION_RSS)
17908                                 return -rte_mtr_error_set(error, ENOTSUP,
17909                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17910                                         NULL, "unsupported action RSS");
17911                         if (!(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17912                                 return -rte_mtr_error_set(error, ENOTSUP,
17913                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17914                                         NULL, "no fate action is found");
17915                 } else {
17916                         if (!(action_flags[i] & MLX5_FLOW_FATE_ACTIONS) &&
17917                             (domain_color[i] & MLX5_MTR_DOMAIN_INGRESS_BIT)) {
17918                                 if ((domain_color[i] &
17919                                      MLX5_MTR_DOMAIN_EGRESS_BIT))
17920                                         domain_color[i] =
17921                                                 MLX5_MTR_DOMAIN_EGRESS_BIT;
17922                                 else
17923                                         return -rte_mtr_error_set(error,
17924                                                 ENOTSUP,
17925                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17926                                                 NULL,
17927                                                 "no fate action is found");
17928                         }
17929                 }
17930         }
17931         /* If both colors have RSS, the attributes should be the same. */
17932         if (flow_dv_mtr_policy_rss_compare(rss_color[RTE_COLOR_GREEN],
17933                                            rss_color[RTE_COLOR_YELLOW]))
17934                 return -rte_mtr_error_set(error, EINVAL,
17935                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17936                                           NULL, "policy RSS attr conflict");
17937         if (rss_color[RTE_COLOR_GREEN] || rss_color[RTE_COLOR_YELLOW])
17938                 *is_rss = true;
17939         /* "domain_color[C]" is non-zero for each color, default is ALL. */
17940         if (!def_green && !def_yellow &&
17941             domain_color[RTE_COLOR_GREEN] != domain_color[RTE_COLOR_YELLOW] &&
17942             !(action_flags[RTE_COLOR_GREEN] & MLX5_FLOW_ACTION_DROP) &&
17943             !(action_flags[RTE_COLOR_YELLOW] & MLX5_FLOW_ACTION_DROP))
17944                 return -rte_mtr_error_set(error, EINVAL,
17945                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17946                                           NULL, "policy domains conflict");
17947         /*
17948          * At least one color policy is listed in the actions, the domains
17949          * to be supported should be the intersection.
17950          */
17951         *domain_bitmap = domain_color[RTE_COLOR_GREEN] &
17952                          domain_color[RTE_COLOR_YELLOW];
17953         return 0;
17954 }
17955
17956 static int
17957 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
17958 {
17959         struct mlx5_priv *priv = dev->data->dev_private;
17960         int ret = 0;
17961
17962         if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
17963                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain,
17964                                                 flags);
17965                 if (ret != 0)
17966                         return ret;
17967         }
17968         if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
17969                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags);
17970                 if (ret != 0)
17971                         return ret;
17972         }
17973         if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
17974                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags);
17975                 if (ret != 0)
17976                         return ret;
17977         }
17978         return 0;
17979 }
17980
17981 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
17982         .validate = flow_dv_validate,
17983         .prepare = flow_dv_prepare,
17984         .translate = flow_dv_translate,
17985         .apply = flow_dv_apply,
17986         .remove = flow_dv_remove,
17987         .destroy = flow_dv_destroy,
17988         .query = flow_dv_query,
17989         .create_mtr_tbls = flow_dv_create_mtr_tbls,
17990         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls,
17991         .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls,
17992         .create_meter = flow_dv_mtr_alloc,
17993         .free_meter = flow_dv_aso_mtr_release_to_pool,
17994         .validate_mtr_acts = flow_dv_validate_mtr_policy_acts,
17995         .create_mtr_acts = flow_dv_create_mtr_policy_acts,
17996         .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts,
17997         .create_policy_rules = flow_dv_create_policy_rules,
17998         .destroy_policy_rules = flow_dv_destroy_policy_rules,
17999         .create_def_policy = flow_dv_create_def_policy,
18000         .destroy_def_policy = flow_dv_destroy_def_policy,
18001         .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare,
18002         .meter_hierarchy_rule_create = flow_dv_meter_hierarchy_rule_create,
18003         .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq,
18004         .counter_alloc = flow_dv_counter_allocate,
18005         .counter_free = flow_dv_counter_free,
18006         .counter_query = flow_dv_counter_query,
18007         .get_aged_flows = flow_dv_get_aged_flows,
18008         .action_validate = flow_dv_action_validate,
18009         .action_create = flow_dv_action_create,
18010         .action_destroy = flow_dv_action_destroy,
18011         .action_update = flow_dv_action_update,
18012         .action_query = flow_dv_action_query,
18013         .sync_domain = flow_dv_sync_domain,
18014 };
18015
18016 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
18017