drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24 #include <rte_mtr.h>
  25 #include <rte_mtr_driver.h>
  26
  27 #include <mlx5_glue.h>
  28 #include <mlx5_devx_cmds.h>
  29 #include <mlx5_prm.h>
  30 #include <mlx5_malloc.h>
  31
  32 #include "mlx5_defs.h"
  33 #include "mlx5.h"
  34 #include "mlx5_common_os.h"
  35 #include "mlx5_flow.h"
  36 #include "mlx5_flow_os.h"
  37 #include "mlx5_rx.h"
  38 #include "mlx5_tx.h"
  39 #include "rte_pmd_mlx5.h"
  40
  41 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  42
  43 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  44 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  45 #endif
  46
  47 #ifndef HAVE_MLX5DV_DR_ESWITCH
  48 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  49 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  50 #endif
  51 #endif
  52
  53 #ifndef HAVE_MLX5DV_DR
  54 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  55 #endif
  56
  57 /* VLAN header definitions */
  58 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  59 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  60 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  61 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  62 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  63
  64 union flow_dv_attr {
  65         struct {
  66                 uint32_t valid:1;
  67                 uint32_t ipv4:1;
  68                 uint32_t ipv6:1;
  69                 uint32_t tcp:1;
  70                 uint32_t udp:1;
  71                 uint32_t reserved:27;
  72         };
  73         uint32_t attr;
  74 };
  75
  76 static int
  77 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
  78                              struct mlx5_flow_tbl_resource *tbl);
  79
  80 static int
  81 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  82                                      uint32_t encap_decap_idx);
  83
  84 static int
  85 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  86                                         uint32_t port_id);
  87 static void
  88 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);
  89
  90 static int
  91 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
  92                                   uint32_t rix_jump);
  93
  94 /**
  95  * Initialize flow attributes structure according to flow items' types.
  96  *
  97  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  98  * mode. For tunnel mode, the items to be modified are the outermost ones.
  99  *
 100  * @param[in] item
 101  *   Pointer to item specification.
 102  * @param[out] attr
 103  *   Pointer to flow attributes structure.
 104  * @param[in] dev_flow
 105  *   Pointer to the sub flow.
 106  * @param[in] tunnel_decap
 107  *   Whether action is after tunnel decapsulation.
 108  */
 109 static void
 110 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 111                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 112 {
 113         uint64_t layers = dev_flow->handle->layers;
 114
 115         /*
 116          * If layers is already initialized, it means this dev_flow is the
 117          * suffix flow, the layers flags is set by the prefix flow. Need to
 118          * use the layer flags from prefix flow as the suffix flow may not
 119          * have the user defined items as the flow is split.
 120          */
 121         if (layers) {
 122                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 123                         attr->ipv4 = 1;
 124                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 125                         attr->ipv6 = 1;
 126                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 127                         attr->tcp = 1;
 128                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 129                         attr->udp = 1;
 130                 attr->valid = 1;
 131                 return;
 132         }
 133         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 134                 uint8_t next_protocol = 0xff;
 135                 switch (item->type) {
 136                 case RTE_FLOW_ITEM_TYPE_GRE:
 137                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 138                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 139                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 140                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 141                 case RTE_FLOW_ITEM_TYPE_MPLS:
 142                         if (tunnel_decap)
 143                                 attr->attr = 0;
 144                         break;
 145                 case RTE_FLOW_ITEM_TYPE_IPV4:
 146                         if (!attr->ipv6)
 147                                 attr->ipv4 = 1;
 148                         if (item->mask != NULL &&
 149                             ((const struct rte_flow_item_ipv4 *)
 150                             item->mask)->hdr.next_proto_id)
 151                                 next_protocol =
 152                                     ((const struct rte_flow_item_ipv4 *)
 153                                       (item->spec))->hdr.next_proto_id &
 154                                     ((const struct rte_flow_item_ipv4 *)
 155                                       (item->mask))->hdr.next_proto_id;
 156                         if ((next_protocol == IPPROTO_IPIP ||
 157                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 158                                 attr->attr = 0;
 159                         break;
 160                 case RTE_FLOW_ITEM_TYPE_IPV6:
 161                         if (!attr->ipv4)
 162                                 attr->ipv6 = 1;
 163                         if (item->mask != NULL &&
 164                             ((const struct rte_flow_item_ipv6 *)
 165                             item->mask)->hdr.proto)
 166                                 next_protocol =
 167                                     ((const struct rte_flow_item_ipv6 *)
 168                                       (item->spec))->hdr.proto &
 169                                     ((const struct rte_flow_item_ipv6 *)
 170                                       (item->mask))->hdr.proto;
 171                         if ((next_protocol == IPPROTO_IPIP ||
 172                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 173                                 attr->attr = 0;
 174                         break;
 175                 case RTE_FLOW_ITEM_TYPE_UDP:
 176                         if (!attr->tcp)
 177                                 attr->udp = 1;
 178                         break;
 179                 case RTE_FLOW_ITEM_TYPE_TCP:
 180                         if (!attr->udp)
 181                                 attr->tcp = 1;
 182                         break;
 183                 default:
 184                         break;
 185                 }
 186         }
 187         attr->valid = 1;
 188 }
 189
 190 /**
 191  * Convert rte_mtr_color to mlx5 color.
 192  *
 193  * @param[in] rcol
 194  *   rte_mtr_color.
 195  *
 196  * @return
 197  *   mlx5 color.
 198  */
 199 static int
 200 rte_col_2_mlx5_col(enum rte_color rcol)
 201 {
 202         switch (rcol) {
 203         case RTE_COLOR_GREEN:
 204                 return MLX5_FLOW_COLOR_GREEN;
 205         case RTE_COLOR_YELLOW:
 206                 return MLX5_FLOW_COLOR_YELLOW;
 207         case RTE_COLOR_RED:
 208                 return MLX5_FLOW_COLOR_RED;
 209         default:
 210                 break;
 211         }
 212         return MLX5_FLOW_COLOR_UNDEFINED;
 213 }
 214
 215 struct field_modify_info {
 216         uint32_t size; /* Size of field in protocol header, in bytes. */
 217         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 218         enum mlx5_modification_field id;
 219 };
 220
 221 struct field_modify_info modify_eth[] = {
 222         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 223         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 224         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 225         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 226         {0, 0, 0},
 227 };
 228
 229 struct field_modify_info modify_vlan_out_first_vid[] = {
 230         /* Size in bits !!! */
 231         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 232         {0, 0, 0},
 233 };
 234
 235 struct field_modify_info modify_ipv4[] = {
 236         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 237         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 238         {4, 12, MLX5_MODI_OUT_SIPV4},
 239         {4, 16, MLX5_MODI_OUT_DIPV4},
 240         {0, 0, 0},
 241 };
 242
 243 struct field_modify_info modify_ipv6[] = {
 244         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 245         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 246         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 247         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 248         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 249         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 250         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 251         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 252         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 253         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 254         {0, 0, 0},
 255 };
 256
 257 struct field_modify_info modify_udp[] = {
 258         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 259         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 260         {0, 0, 0},
 261 };
 262
 263 struct field_modify_info modify_tcp[] = {
 264         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 265         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 266         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 267         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 268         {0, 0, 0},
 269 };
 270
 271 static const struct rte_flow_item *
 272 mlx5_flow_find_tunnel_item(const struct rte_flow_item *item)
 273 {
 274         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 275                 switch (item->type) {
 276                 default:
 277                         break;
 278                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 279                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 280                 case RTE_FLOW_ITEM_TYPE_GRE:
 281                 case RTE_FLOW_ITEM_TYPE_MPLS:
 282                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 283                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 284                         return item;
 285                 case RTE_FLOW_ITEM_TYPE_IPV4:
 286                 case RTE_FLOW_ITEM_TYPE_IPV6:
 287                         if (item[1].type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 288                             item[1].type == RTE_FLOW_ITEM_TYPE_IPV6)
 289                                 return item;
 290                         break;
 291                 }
 292         }
 293         return NULL;
 294 }
 295
 296 static void
 297 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 298                           uint8_t next_protocol, uint64_t *item_flags,
 299                           int *tunnel)
 300 {
 301         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 302                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 303         if (next_protocol == IPPROTO_IPIP) {
 304                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 305                 *tunnel = 1;
 306         }
 307         if (next_protocol == IPPROTO_IPV6) {
 308                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 309                 *tunnel = 1;
 310         }
 311 }
 312
 313 /* Update VLAN's VID/PCP based on input rte_flow_action.
 314  *
 315  * @param[in] action
 316  *   Pointer to struct rte_flow_action.
 317  * @param[out] vlan
 318  *   Pointer to struct rte_vlan_hdr.
 319  */
 320 static void
 321 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 322                          struct rte_vlan_hdr *vlan)
 323 {
 324         uint16_t vlan_tci;
 325         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 326                 vlan_tci =
 327                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 328                                                action->conf)->vlan_pcp;
 329                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 330                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 331                 vlan->vlan_tci |= vlan_tci;
 332         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 333                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 334                 vlan->vlan_tci |= rte_be_to_cpu_16
 335                     (((const struct rte_flow_action_of_set_vlan_vid *)
 336                                              action->conf)->vlan_vid);
 337         }
 338 }
 339
 340 /**
 341  * Fetch 1, 2, 3 or 4 byte field from the byte array
 342  * and return as unsigned integer in host-endian format.
 343  *
 344  * @param[in] data
 345  *   Pointer to data array.
 346  * @param[in] size
 347  *   Size of field to extract.
 348  *
 349  * @return
 350  *   converted field in host endian format.
 351  */
 352 static inline uint32_t
 353 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 354 {
 355         uint32_t ret;
 356
 357         switch (size) {
 358         case 1:
 359                 ret = *data;
 360                 break;
 361         case 2:
 362                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 363                 break;
 364         case 3:
 365                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 366                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 367                 break;
 368         case 4:
 369                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 370                 break;
 371         default:
 372                 MLX5_ASSERT(false);
 373                 ret = 0;
 374                 break;
 375         }
 376         return ret;
 377 }
 378
 379 /**
 380  * Convert modify-header action to DV specification.
 381  *
 382  * Data length of each action is determined by provided field description
 383  * and the item mask. Data bit offset and width of each action is determined
 384  * by provided item mask.
 385  *
 386  * @param[in] item
 387  *   Pointer to item specification.
 388  * @param[in] field
 389  *   Pointer to field modification information.
 390  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 391  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 392  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 393  * @param[in] dcopy
 394  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 395  *   Negative offset value sets the same offset as source offset.
 396  *   size field is ignored, value is taken from source field.
 397  * @param[in,out] resource
 398  *   Pointer to the modify-header resource.
 399  * @param[in] type
 400  *   Type of modification.
 401  * @param[out] error
 402  *   Pointer to the error structure.
 403  *
 404  * @return
 405  *   0 on success, a negative errno value otherwise and rte_errno is set.
 406  */
 407 static int
 408 flow_dv_convert_modify_action(struct rte_flow_item *item,
 409                               struct field_modify_info *field,
 410                               struct field_modify_info *dcopy,
 411                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 412                               uint32_t type, struct rte_flow_error *error)
 413 {
 414         uint32_t i = resource->actions_num;
 415         struct mlx5_modification_cmd *actions = resource->actions;
 416         uint32_t carry_b = 0;
 417
 418         /*
 419          * The item and mask are provided in big-endian format.
 420          * The fields should be presented as in big-endian format either.
 421          * Mask must be always present, it defines the actual field width.
 422          */
 423         MLX5_ASSERT(item->mask);
 424         MLX5_ASSERT(field->size);
 425         do {
 426                 uint32_t size_b;
 427                 uint32_t off_b;
 428                 uint32_t mask;
 429                 uint32_t data;
 430                 bool next_field = true;
 431                 bool next_dcopy = true;
 432
 433                 if (i >= MLX5_MAX_MODIFY_NUM)
 434                         return rte_flow_error_set(error, EINVAL,
 435                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 436                                  "too many items to modify");
 437                 /* Fetch variable byte size mask from the array. */
 438                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 439                                            field->offset, field->size);
 440                 if (!mask) {
 441                         ++field;
 442                         continue;
 443                 }
 444                 /* Deduce actual data width in bits from mask value. */
 445                 off_b = rte_bsf32(mask) + carry_b;
 446                 size_b = sizeof(uint32_t) * CHAR_BIT -
 447                          off_b - __builtin_clz(mask);
 448                 MLX5_ASSERT(size_b);
 449                 actions[i] = (struct mlx5_modification_cmd) {
 450                         .action_type = type,
 451                         .field = field->id,
 452                         .offset = off_b,
 453                         .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ?
 454                                 0 : size_b,
 455                 };
 456                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 457                         MLX5_ASSERT(dcopy);
 458                         actions[i].dst_field = dcopy->id;
 459                         actions[i].dst_offset =
 460                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 461                         /* Convert entire record to big-endian format. */
 462                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 463                         /*
 464                          * Destination field overflow. Copy leftovers of
 465                          * a source field to the next destination field.
 466                          */
 467                         carry_b = 0;
 468                         if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) &&
 469                             dcopy->size != 0) {
 470                                 actions[i].length =
 471                                         dcopy->size * CHAR_BIT - dcopy->offset;
 472                                 carry_b = actions[i].length;
 473                                 next_field = false;
 474                         }
 475                         /*
 476                          * Not enough bits in a source filed to fill a
 477                          * destination field. Switch to the next source.
 478                          */
 479                         if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) &&
 480                             (size_b == field->size * CHAR_BIT - off_b)) {
 481                                 actions[i].length =
 482                                         field->size * CHAR_BIT - off_b;
 483                                 dcopy->offset += actions[i].length;
 484                                 next_dcopy = false;
 485                         }
 486                         if (next_dcopy)
 487                                 ++dcopy;
 488                 } else {
 489                         MLX5_ASSERT(item->spec);
 490                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 491                                                    field->offset, field->size);
 492                         /* Shift out the trailing masked bits from data. */
 493                         data = (data & mask) >> off_b;
 494                         actions[i].data1 = rte_cpu_to_be_32(data);
 495                 }
 496                 /* Convert entire record to expected big-endian format. */
 497                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 498                 if (next_field)
 499                         ++field;
 500                 ++i;
 501         } while (field->size);
 502         if (resource->actions_num == i)
 503                 return rte_flow_error_set(error, EINVAL,
 504                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 505                                           "invalid modification flow item");
 506         resource->actions_num = i;
 507         return 0;
 508 }
 509
 510 /**
 511  * Convert modify-header set IPv4 address action to DV specification.
 512  *
 513  * @param[in,out] resource
 514  *   Pointer to the modify-header resource.
 515  * @param[in] action
 516  *   Pointer to action specification.
 517  * @param[out] error
 518  *   Pointer to the error structure.
 519  *
 520  * @return
 521  *   0 on success, a negative errno value otherwise and rte_errno is set.
 522  */
 523 static int
 524 flow_dv_convert_action_modify_ipv4
 525                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 526                          const struct rte_flow_action *action,
 527                          struct rte_flow_error *error)
 528 {
 529         const struct rte_flow_action_set_ipv4 *conf =
 530                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 531         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 532         struct rte_flow_item_ipv4 ipv4;
 533         struct rte_flow_item_ipv4 ipv4_mask;
 534
 535         memset(&ipv4, 0, sizeof(ipv4));
 536         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 537         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 538                 ipv4.hdr.src_addr = conf->ipv4_addr;
 539                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 540         } else {
 541                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 542                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 543         }
 544         item.spec = &ipv4;
 545         item.mask = &ipv4_mask;
 546         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 547                                              MLX5_MODIFICATION_TYPE_SET, error);
 548 }
 549
 550 /**
 551  * Convert modify-header set IPv6 address action to DV specification.
 552  *
 553  * @param[in,out] resource
 554  *   Pointer to the modify-header resource.
 555  * @param[in] action
 556  *   Pointer to action specification.
 557  * @param[out] error
 558  *   Pointer to the error structure.
 559  *
 560  * @return
 561  *   0 on success, a negative errno value otherwise and rte_errno is set.
 562  */
 563 static int
 564 flow_dv_convert_action_modify_ipv6
 565                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 566                          const struct rte_flow_action *action,
 567                          struct rte_flow_error *error)
 568 {
 569         const struct rte_flow_action_set_ipv6 *conf =
 570                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 571         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 572         struct rte_flow_item_ipv6 ipv6;
 573         struct rte_flow_item_ipv6 ipv6_mask;
 574
 575         memset(&ipv6, 0, sizeof(ipv6));
 576         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 577         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 578                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 579                        sizeof(ipv6.hdr.src_addr));
 580                 memcpy(&ipv6_mask.hdr.src_addr,
 581                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 582                        sizeof(ipv6.hdr.src_addr));
 583         } else {
 584                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 585                        sizeof(ipv6.hdr.dst_addr));
 586                 memcpy(&ipv6_mask.hdr.dst_addr,
 587                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 588                        sizeof(ipv6.hdr.dst_addr));
 589         }
 590         item.spec = &ipv6;
 591         item.mask = &ipv6_mask;
 592         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 593                                              MLX5_MODIFICATION_TYPE_SET, error);
 594 }
 595
 596 /**
 597  * Convert modify-header set MAC address action to DV specification.
 598  *
 599  * @param[in,out] resource
 600  *   Pointer to the modify-header resource.
 601  * @param[in] action
 602  *   Pointer to action specification.
 603  * @param[out] error
 604  *   Pointer to the error structure.
 605  *
 606  * @return
 607  *   0 on success, a negative errno value otherwise and rte_errno is set.
 608  */
 609 static int
 610 flow_dv_convert_action_modify_mac
 611                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 612                          const struct rte_flow_action *action,
 613                          struct rte_flow_error *error)
 614 {
 615         const struct rte_flow_action_set_mac *conf =
 616                 (const struct rte_flow_action_set_mac *)(action->conf);
 617         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 618         struct rte_flow_item_eth eth;
 619         struct rte_flow_item_eth eth_mask;
 620
 621         memset(&eth, 0, sizeof(eth));
 622         memset(&eth_mask, 0, sizeof(eth_mask));
 623         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 624                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 625                        sizeof(eth.src.addr_bytes));
 626                 memcpy(&eth_mask.src.addr_bytes,
 627                        &rte_flow_item_eth_mask.src.addr_bytes,
 628                        sizeof(eth_mask.src.addr_bytes));
 629         } else {
 630                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 631                        sizeof(eth.dst.addr_bytes));
 632                 memcpy(&eth_mask.dst.addr_bytes,
 633                        &rte_flow_item_eth_mask.dst.addr_bytes,
 634                        sizeof(eth_mask.dst.addr_bytes));
 635         }
 636         item.spec = &eth;
 637         item.mask = &eth_mask;
 638         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 639                                              MLX5_MODIFICATION_TYPE_SET, error);
 640 }
 641
 642 /**
 643  * Convert modify-header set VLAN VID action to DV specification.
 644  *
 645  * @param[in,out] resource
 646  *   Pointer to the modify-header resource.
 647  * @param[in] action
 648  *   Pointer to action specification.
 649  * @param[out] error
 650  *   Pointer to the error structure.
 651  *
 652  * @return
 653  *   0 on success, a negative errno value otherwise and rte_errno is set.
 654  */
 655 static int
 656 flow_dv_convert_action_modify_vlan_vid
 657                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 658                          const struct rte_flow_action *action,
 659                          struct rte_flow_error *error)
 660 {
 661         const struct rte_flow_action_of_set_vlan_vid *conf =
 662                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 663         int i = resource->actions_num;
 664         struct mlx5_modification_cmd *actions = resource->actions;
 665         struct field_modify_info *field = modify_vlan_out_first_vid;
 666
 667         if (i >= MLX5_MAX_MODIFY_NUM)
 668                 return rte_flow_error_set(error, EINVAL,
 669                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 670                          "too many items to modify");
 671         actions[i] = (struct mlx5_modification_cmd) {
 672                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 673                 .field = field->id,
 674                 .length = field->size,
 675                 .offset = field->offset,
 676         };
 677         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 678         actions[i].data1 = conf->vlan_vid;
 679         actions[i].data1 = actions[i].data1 << 16;
 680         resource->actions_num = ++i;
 681         return 0;
 682 }
 683
 684 /**
 685  * Convert modify-header set TP action to DV specification.
 686  *
 687  * @param[in,out] resource
 688  *   Pointer to the modify-header resource.
 689  * @param[in] action
 690  *   Pointer to action specification.
 691  * @param[in] items
 692  *   Pointer to rte_flow_item objects list.
 693  * @param[in] attr
 694  *   Pointer to flow attributes structure.
 695  * @param[in] dev_flow
 696  *   Pointer to the sub flow.
 697  * @param[in] tunnel_decap
 698  *   Whether action is after tunnel decapsulation.
 699  * @param[out] error
 700  *   Pointer to the error structure.
 701  *
 702  * @return
 703  *   0 on success, a negative errno value otherwise and rte_errno is set.
 704  */
 705 static int
 706 flow_dv_convert_action_modify_tp
 707                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 708                          const struct rte_flow_action *action,
 709                          const struct rte_flow_item *items,
 710                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 711                          bool tunnel_decap, struct rte_flow_error *error)
 712 {
 713         const struct rte_flow_action_set_tp *conf =
 714                 (const struct rte_flow_action_set_tp *)(action->conf);
 715         struct rte_flow_item item;
 716         struct rte_flow_item_udp udp;
 717         struct rte_flow_item_udp udp_mask;
 718         struct rte_flow_item_tcp tcp;
 719         struct rte_flow_item_tcp tcp_mask;
 720         struct field_modify_info *field;
 721
 722         if (!attr->valid)
 723                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 724         if (attr->udp) {
 725                 memset(&udp, 0, sizeof(udp));
 726                 memset(&udp_mask, 0, sizeof(udp_mask));
 727                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 728                         udp.hdr.src_port = conf->port;
 729                         udp_mask.hdr.src_port =
 730                                         rte_flow_item_udp_mask.hdr.src_port;
 731                 } else {
 732                         udp.hdr.dst_port = conf->port;
 733                         udp_mask.hdr.dst_port =
 734                                         rte_flow_item_udp_mask.hdr.dst_port;
 735                 }
 736                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 737                 item.spec = &udp;
 738                 item.mask = &udp_mask;
 739                 field = modify_udp;
 740         } else {
 741                 MLX5_ASSERT(attr->tcp);
 742                 memset(&tcp, 0, sizeof(tcp));
 743                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 744                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 745                         tcp.hdr.src_port = conf->port;
 746                         tcp_mask.hdr.src_port =
 747                                         rte_flow_item_tcp_mask.hdr.src_port;
 748                 } else {
 749                         tcp.hdr.dst_port = conf->port;
 750                         tcp_mask.hdr.dst_port =
 751                                         rte_flow_item_tcp_mask.hdr.dst_port;
 752                 }
 753                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 754                 item.spec = &tcp;
 755                 item.mask = &tcp_mask;
 756                 field = modify_tcp;
 757         }
 758         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 759                                              MLX5_MODIFICATION_TYPE_SET, error);
 760 }
 761
 762 /**
 763  * Convert modify-header set TTL action to DV specification.
 764  *
 765  * @param[in,out] resource
 766  *   Pointer to the modify-header resource.
 767  * @param[in] action
 768  *   Pointer to action specification.
 769  * @param[in] items
 770  *   Pointer to rte_flow_item objects list.
 771  * @param[in] attr
 772  *   Pointer to flow attributes structure.
 773  * @param[in] dev_flow
 774  *   Pointer to the sub flow.
 775  * @param[in] tunnel_decap
 776  *   Whether action is after tunnel decapsulation.
 777  * @param[out] error
 778  *   Pointer to the error structure.
 779  *
 780  * @return
 781  *   0 on success, a negative errno value otherwise and rte_errno is set.
 782  */
 783 static int
 784 flow_dv_convert_action_modify_ttl
 785                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 786                          const struct rte_flow_action *action,
 787                          const struct rte_flow_item *items,
 788                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 789                          bool tunnel_decap, struct rte_flow_error *error)
 790 {
 791         const struct rte_flow_action_set_ttl *conf =
 792                 (const struct rte_flow_action_set_ttl *)(action->conf);
 793         struct rte_flow_item item;
 794         struct rte_flow_item_ipv4 ipv4;
 795         struct rte_flow_item_ipv4 ipv4_mask;
 796         struct rte_flow_item_ipv6 ipv6;
 797         struct rte_flow_item_ipv6 ipv6_mask;
 798         struct field_modify_info *field;
 799
 800         if (!attr->valid)
 801                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 802         if (attr->ipv4) {
 803                 memset(&ipv4, 0, sizeof(ipv4));
 804                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 805                 ipv4.hdr.time_to_live = conf->ttl_value;
 806                 ipv4_mask.hdr.time_to_live = 0xFF;
 807                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 808                 item.spec = &ipv4;
 809                 item.mask = &ipv4_mask;
 810                 field = modify_ipv4;
 811         } else {
 812                 MLX5_ASSERT(attr->ipv6);
 813                 memset(&ipv6, 0, sizeof(ipv6));
 814                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 815                 ipv6.hdr.hop_limits = conf->ttl_value;
 816                 ipv6_mask.hdr.hop_limits = 0xFF;
 817                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 818                 item.spec = &ipv6;
 819                 item.mask = &ipv6_mask;
 820                 field = modify_ipv6;
 821         }
 822         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 823                                              MLX5_MODIFICATION_TYPE_SET, error);
 824 }
 825
 826 /**
 827  * Convert modify-header decrement TTL action to DV specification.
 828  *
 829  * @param[in,out] resource
 830  *   Pointer to the modify-header resource.
 831  * @param[in] action
 832  *   Pointer to action specification.
 833  * @param[in] items
 834  *   Pointer to rte_flow_item objects list.
 835  * @param[in] attr
 836  *   Pointer to flow attributes structure.
 837  * @param[in] dev_flow
 838  *   Pointer to the sub flow.
 839  * @param[in] tunnel_decap
 840  *   Whether action is after tunnel decapsulation.
 841  * @param[out] error
 842  *   Pointer to the error structure.
 843  *
 844  * @return
 845  *   0 on success, a negative errno value otherwise and rte_errno is set.
 846  */
 847 static int
 848 flow_dv_convert_action_modify_dec_ttl
 849                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 850                          const struct rte_flow_item *items,
 851                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 852                          bool tunnel_decap, struct rte_flow_error *error)
 853 {
 854         struct rte_flow_item item;
 855         struct rte_flow_item_ipv4 ipv4;
 856         struct rte_flow_item_ipv4 ipv4_mask;
 857         struct rte_flow_item_ipv6 ipv6;
 858         struct rte_flow_item_ipv6 ipv6_mask;
 859         struct field_modify_info *field;
 860
 861         if (!attr->valid)
 862                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 863         if (attr->ipv4) {
 864                 memset(&ipv4, 0, sizeof(ipv4));
 865                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 866                 ipv4.hdr.time_to_live = 0xFF;
 867                 ipv4_mask.hdr.time_to_live = 0xFF;
 868                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 869                 item.spec = &ipv4;
 870                 item.mask = &ipv4_mask;
 871                 field = modify_ipv4;
 872         } else {
 873                 MLX5_ASSERT(attr->ipv6);
 874                 memset(&ipv6, 0, sizeof(ipv6));
 875                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 876                 ipv6.hdr.hop_limits = 0xFF;
 877                 ipv6_mask.hdr.hop_limits = 0xFF;
 878                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 879                 item.spec = &ipv6;
 880                 item.mask = &ipv6_mask;
 881                 field = modify_ipv6;
 882         }
 883         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 884                                              MLX5_MODIFICATION_TYPE_ADD, error);
 885 }
 886
 887 /**
 888  * Convert modify-header increment/decrement TCP Sequence number
 889  * to DV specification.
 890  *
 891  * @param[in,out] resource
 892  *   Pointer to the modify-header resource.
 893  * @param[in] action
 894  *   Pointer to action specification.
 895  * @param[out] error
 896  *   Pointer to the error structure.
 897  *
 898  * @return
 899  *   0 on success, a negative errno value otherwise and rte_errno is set.
 900  */
 901 static int
 902 flow_dv_convert_action_modify_tcp_seq
 903                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 904                          const struct rte_flow_action *action,
 905                          struct rte_flow_error *error)
 906 {
 907         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 908         uint64_t value = rte_be_to_cpu_32(*conf);
 909         struct rte_flow_item item;
 910         struct rte_flow_item_tcp tcp;
 911         struct rte_flow_item_tcp tcp_mask;
 912
 913         memset(&tcp, 0, sizeof(tcp));
 914         memset(&tcp_mask, 0, sizeof(tcp_mask));
 915         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 916                 /*
 917                  * The HW has no decrement operation, only increment operation.
 918                  * To simulate decrement X from Y using increment operation
 919                  * we need to add UINT32_MAX X times to Y.
 920                  * Each adding of UINT32_MAX decrements Y by 1.
 921                  */
 922                 value *= UINT32_MAX;
 923         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 924         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 925         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 926         item.spec = &tcp;
 927         item.mask = &tcp_mask;
 928         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 929                                              MLX5_MODIFICATION_TYPE_ADD, error);
 930 }
 931
 932 /**
 933  * Convert modify-header increment/decrement TCP Acknowledgment number
 934  * to DV specification.
 935  *
 936  * @param[in,out] resource
 937  *   Pointer to the modify-header resource.
 938  * @param[in] action
 939  *   Pointer to action specification.
 940  * @param[out] error
 941  *   Pointer to the error structure.
 942  *
 943  * @return
 944  *   0 on success, a negative errno value otherwise and rte_errno is set.
 945  */
 946 static int
 947 flow_dv_convert_action_modify_tcp_ack
 948                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 949                          const struct rte_flow_action *action,
 950                          struct rte_flow_error *error)
 951 {
 952         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 953         uint64_t value = rte_be_to_cpu_32(*conf);
 954         struct rte_flow_item item;
 955         struct rte_flow_item_tcp tcp;
 956         struct rte_flow_item_tcp tcp_mask;
 957
 958         memset(&tcp, 0, sizeof(tcp));
 959         memset(&tcp_mask, 0, sizeof(tcp_mask));
 960         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 961                 /*
 962                  * The HW has no decrement operation, only increment operation.
 963                  * To simulate decrement X from Y using increment operation
 964                  * we need to add UINT32_MAX X times to Y.
 965                  * Each adding of UINT32_MAX decrements Y by 1.
 966                  */
 967                 value *= UINT32_MAX;
 968         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 969         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 970         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 971         item.spec = &tcp;
 972         item.mask = &tcp_mask;
 973         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 974                                              MLX5_MODIFICATION_TYPE_ADD, error);
 975 }
 976
 977 static enum mlx5_modification_field reg_to_field[] = {
 978         [REG_NON] = MLX5_MODI_OUT_NONE,
 979         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 980         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 981         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 982         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 983         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 984         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 985         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 986         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 987         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 988         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 989 };
 990
 991 /**
 992  * Convert register set to DV specification.
 993  *
 994  * @param[in,out] resource
 995  *   Pointer to the modify-header resource.
 996  * @param[in] action
 997  *   Pointer to action specification.
 998  * @param[out] error
 999  *   Pointer to the error structure.
1000  *
1001  * @return
1002  *   0 on success, a negative errno value otherwise and rte_errno is set.
1003  */
1004 static int
1005 flow_dv_convert_action_set_reg
1006                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1007                          const struct rte_flow_action *action,
1008                          struct rte_flow_error *error)
1009 {
1010         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
1011         struct mlx5_modification_cmd *actions = resource->actions;
1012         uint32_t i = resource->actions_num;
1013
1014         if (i >= MLX5_MAX_MODIFY_NUM)
1015                 return rte_flow_error_set(error, EINVAL,
1016                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1017                                           "too many items to modify");
1018         MLX5_ASSERT(conf->id != REG_NON);
1019         MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
1020         actions[i] = (struct mlx5_modification_cmd) {
1021                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1022                 .field = reg_to_field[conf->id],
1023                 .offset = conf->offset,
1024                 .length = conf->length,
1025         };
1026         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1027         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1028         ++i;
1029         resource->actions_num = i;
1030         return 0;
1031 }
1032
1033 /**
1034  * Convert SET_TAG action to DV specification.
1035  *
1036  * @param[in] dev
1037  *   Pointer to the rte_eth_dev structure.
1038  * @param[in,out] resource
1039  *   Pointer to the modify-header resource.
1040  * @param[in] conf
1041  *   Pointer to action specification.
1042  * @param[out] error
1043  *   Pointer to the error structure.
1044  *
1045  * @return
1046  *   0 on success, a negative errno value otherwise and rte_errno is set.
1047  */
1048 static int
1049 flow_dv_convert_action_set_tag
1050                         (struct rte_eth_dev *dev,
1051                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1052                          const struct rte_flow_action_set_tag *conf,
1053                          struct rte_flow_error *error)
1054 {
1055         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1056         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1057         struct rte_flow_item item = {
1058                 .spec = &data,
1059                 .mask = &mask,
1060         };
1061         struct field_modify_info reg_c_x[] = {
1062                 [1] = {0, 0, 0},
1063         };
1064         enum mlx5_modification_field reg_type;
1065         int ret;
1066
1067         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1068         if (ret < 0)
1069                 return ret;
1070         MLX5_ASSERT(ret != REG_NON);
1071         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1072         reg_type = reg_to_field[ret];
1073         MLX5_ASSERT(reg_type > 0);
1074         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1075         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1076                                              MLX5_MODIFICATION_TYPE_SET, error);
1077 }
1078
1079 /**
1080  * Convert internal COPY_REG action to DV specification.
1081  *
1082  * @param[in] dev
1083  *   Pointer to the rte_eth_dev structure.
1084  * @param[in,out] res
1085  *   Pointer to the modify-header resource.
1086  * @param[in] action
1087  *   Pointer to action specification.
1088  * @param[out] error
1089  *   Pointer to the error structure.
1090  *
1091  * @return
1092  *   0 on success, a negative errno value otherwise and rte_errno is set.
1093  */
1094 static int
1095 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1096                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1097                                  const struct rte_flow_action *action,
1098                                  struct rte_flow_error *error)
1099 {
1100         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1101         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1102         struct rte_flow_item item = {
1103                 .spec = NULL,
1104                 .mask = &mask,
1105         };
1106         struct field_modify_info reg_src[] = {
1107                 {4, 0, reg_to_field[conf->src]},
1108                 {0, 0, 0},
1109         };
1110         struct field_modify_info reg_dst = {
1111                 .offset = 0,
1112                 .id = reg_to_field[conf->dst],
1113         };
1114         /* Adjust reg_c[0] usage according to reported mask. */
1115         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1116                 struct mlx5_priv *priv = dev->data->dev_private;
1117                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1118
1119                 MLX5_ASSERT(reg_c0);
1120                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1121                 if (conf->dst == REG_C_0) {
1122                         /* Copy to reg_c[0], within mask only. */
1123                         reg_dst.offset = rte_bsf32(reg_c0);
1124                         /*
1125                          * Mask is ignoring the enianness, because
1126                          * there is no conversion in datapath.
1127                          */
1128 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1129                         /* Copy from destination lower bits to reg_c[0]. */
1130                         mask = reg_c0 >> reg_dst.offset;
1131 #else
1132                         /* Copy from destination upper bits to reg_c[0]. */
1133                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1134                                           rte_fls_u32(reg_c0));
1135 #endif
1136                 } else {
1137                         mask = rte_cpu_to_be_32(reg_c0);
1138 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1139                         /* Copy from reg_c[0] to destination lower bits. */
1140                         reg_dst.offset = 0;
1141 #else
1142                         /* Copy from reg_c[0] to destination upper bits. */
1143                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1144                                          (rte_fls_u32(reg_c0) -
1145                                           rte_bsf32(reg_c0));
1146 #endif
1147                 }
1148         }
1149         return flow_dv_convert_modify_action(&item,
1150                                              reg_src, &reg_dst, res,
1151                                              MLX5_MODIFICATION_TYPE_COPY,
1152                                              error);
1153 }
1154
1155 /**
1156  * Convert MARK action to DV specification. This routine is used
1157  * in extensive metadata only and requires metadata register to be
1158  * handled. In legacy mode hardware tag resource is engaged.
1159  *
1160  * @param[in] dev
1161  *   Pointer to the rte_eth_dev structure.
1162  * @param[in] conf
1163  *   Pointer to MARK action specification.
1164  * @param[in,out] resource
1165  *   Pointer to the modify-header resource.
1166  * @param[out] error
1167  *   Pointer to the error structure.
1168  *
1169  * @return
1170  *   0 on success, a negative errno value otherwise and rte_errno is set.
1171  */
1172 static int
1173 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1174                             const struct rte_flow_action_mark *conf,
1175                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1176                             struct rte_flow_error *error)
1177 {
1178         struct mlx5_priv *priv = dev->data->dev_private;
1179         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1180                                            priv->sh->dv_mark_mask);
1181         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1182         struct rte_flow_item item = {
1183                 .spec = &data,
1184                 .mask = &mask,
1185         };
1186         struct field_modify_info reg_c_x[] = {
1187                 [1] = {0, 0, 0},
1188         };
1189         int reg;
1190
1191         if (!mask)
1192                 return rte_flow_error_set(error, EINVAL,
1193                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1194                                           NULL, "zero mark action mask");
1195         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1196         if (reg < 0)
1197                 return reg;
1198         MLX5_ASSERT(reg > 0);
1199         if (reg == REG_C_0) {
1200                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1201                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1202
1203                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1204                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1205                 mask = rte_cpu_to_be_32(mask << shl_c0);
1206         }
1207         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1208         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1209                                              MLX5_MODIFICATION_TYPE_SET, error);
1210 }
1211
1212 /**
1213  * Get metadata register index for specified steering domain.
1214  *
1215  * @param[in] dev
1216  *   Pointer to the rte_eth_dev structure.
1217  * @param[in] attr
1218  *   Attributes of flow to determine steering domain.
1219  * @param[out] error
1220  *   Pointer to the error structure.
1221  *
1222  * @return
1223  *   positive index on success, a negative errno value otherwise
1224  *   and rte_errno is set.
1225  */
1226 static enum modify_reg
1227 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1228                          const struct rte_flow_attr *attr,
1229                          struct rte_flow_error *error)
1230 {
1231         int reg =
1232                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1233                                           MLX5_METADATA_FDB :
1234                                             attr->egress ?
1235                                             MLX5_METADATA_TX :
1236                                             MLX5_METADATA_RX, 0, error);
1237         if (reg < 0)
1238                 return rte_flow_error_set(error,
1239                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1240                                           NULL, "unavailable "
1241                                           "metadata register");
1242         return reg;
1243 }
1244
1245 /**
1246  * Convert SET_META action to DV specification.
1247  *
1248  * @param[in] dev
1249  *   Pointer to the rte_eth_dev structure.
1250  * @param[in,out] resource
1251  *   Pointer to the modify-header resource.
1252  * @param[in] attr
1253  *   Attributes of flow that includes this item.
1254  * @param[in] conf
1255  *   Pointer to action specification.
1256  * @param[out] error
1257  *   Pointer to the error structure.
1258  *
1259  * @return
1260  *   0 on success, a negative errno value otherwise and rte_errno is set.
1261  */
1262 static int
1263 flow_dv_convert_action_set_meta
1264                         (struct rte_eth_dev *dev,
1265                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1266                          const struct rte_flow_attr *attr,
1267                          const struct rte_flow_action_set_meta *conf,
1268                          struct rte_flow_error *error)
1269 {
1270         uint32_t mask = rte_cpu_to_be_32(conf->mask);
1271         uint32_t data = rte_cpu_to_be_32(conf->data) & mask;
1272         struct rte_flow_item item = {
1273                 .spec = &data,
1274                 .mask = &mask,
1275         };
1276         struct field_modify_info reg_c_x[] = {
1277                 [1] = {0, 0, 0},
1278         };
1279         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1280
1281         if (reg < 0)
1282                 return reg;
1283         MLX5_ASSERT(reg != REG_NON);
1284         if (reg == REG_C_0) {
1285                 struct mlx5_priv *priv = dev->data->dev_private;
1286                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1287                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1288
1289                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1290                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1291                 mask = rte_cpu_to_be_32(mask << shl_c0);
1292         }
1293         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1294         /* The routine expects parameters in memory as big-endian ones. */
1295         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1296                                              MLX5_MODIFICATION_TYPE_SET, error);
1297 }
1298
1299 /**
1300  * Convert modify-header set IPv4 DSCP action to DV specification.
1301  *
1302  * @param[in,out] resource
1303  *   Pointer to the modify-header resource.
1304  * @param[in] action
1305  *   Pointer to action specification.
1306  * @param[out] error
1307  *   Pointer to the error structure.
1308  *
1309  * @return
1310  *   0 on success, a negative errno value otherwise and rte_errno is set.
1311  */
1312 static int
1313 flow_dv_convert_action_modify_ipv4_dscp
1314                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1315                          const struct rte_flow_action *action,
1316                          struct rte_flow_error *error)
1317 {
1318         const struct rte_flow_action_set_dscp *conf =
1319                 (const struct rte_flow_action_set_dscp *)(action->conf);
1320         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1321         struct rte_flow_item_ipv4 ipv4;
1322         struct rte_flow_item_ipv4 ipv4_mask;
1323
1324         memset(&ipv4, 0, sizeof(ipv4));
1325         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1326         ipv4.hdr.type_of_service = conf->dscp;
1327         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1328         item.spec = &ipv4;
1329         item.mask = &ipv4_mask;
1330         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1331                                              MLX5_MODIFICATION_TYPE_SET, error);
1332 }
1333
1334 /**
1335  * Convert modify-header set IPv6 DSCP action to DV specification.
1336  *
1337  * @param[in,out] resource
1338  *   Pointer to the modify-header resource.
1339  * @param[in] action
1340  *   Pointer to action specification.
1341  * @param[out] error
1342  *   Pointer to the error structure.
1343  *
1344  * @return
1345  *   0 on success, a negative errno value otherwise and rte_errno is set.
1346  */
1347 static int
1348 flow_dv_convert_action_modify_ipv6_dscp
1349                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1350                          const struct rte_flow_action *action,
1351                          struct rte_flow_error *error)
1352 {
1353         const struct rte_flow_action_set_dscp *conf =
1354                 (const struct rte_flow_action_set_dscp *)(action->conf);
1355         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1356         struct rte_flow_item_ipv6 ipv6;
1357         struct rte_flow_item_ipv6 ipv6_mask;
1358
1359         memset(&ipv6, 0, sizeof(ipv6));
1360         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1361         /*
1362          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1363          * rdma-core only accept the DSCP bits byte aligned start from
1364          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1365          * bits in IPv6 case as rdma-core requires byte aligned value.
1366          */
1367         ipv6.hdr.vtc_flow = conf->dscp;
1368         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1369         item.spec = &ipv6;
1370         item.mask = &ipv6_mask;
1371         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1372                                              MLX5_MODIFICATION_TYPE_SET, error);
1373 }
1374
1375 static int
1376 mlx5_flow_item_field_width(struct mlx5_dev_config *config,
1377                            enum rte_flow_field_id field)
1378 {
1379         switch (field) {
1380         case RTE_FLOW_FIELD_START:
1381                 return 32;
1382         case RTE_FLOW_FIELD_MAC_DST:
1383         case RTE_FLOW_FIELD_MAC_SRC:
1384                 return 48;
1385         case RTE_FLOW_FIELD_VLAN_TYPE:
1386                 return 16;
1387         case RTE_FLOW_FIELD_VLAN_ID:
1388                 return 12;
1389         case RTE_FLOW_FIELD_MAC_TYPE:
1390                 return 16;
1391         case RTE_FLOW_FIELD_IPV4_DSCP:
1392                 return 6;
1393         case RTE_FLOW_FIELD_IPV4_TTL:
1394                 return 8;
1395         case RTE_FLOW_FIELD_IPV4_SRC:
1396         case RTE_FLOW_FIELD_IPV4_DST:
1397                 return 32;
1398         case RTE_FLOW_FIELD_IPV6_DSCP:
1399                 return 6;
1400         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1401                 return 8;
1402         case RTE_FLOW_FIELD_IPV6_SRC:
1403         case RTE_FLOW_FIELD_IPV6_DST:
1404                 return 128;
1405         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1406         case RTE_FLOW_FIELD_TCP_PORT_DST:
1407                 return 16;
1408         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1409         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1410                 return 32;
1411         case RTE_FLOW_FIELD_TCP_FLAGS:
1412                 return 9;
1413         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1414         case RTE_FLOW_FIELD_UDP_PORT_DST:
1415                 return 16;
1416         case RTE_FLOW_FIELD_VXLAN_VNI:
1417         case RTE_FLOW_FIELD_GENEVE_VNI:
1418                 return 24;
1419         case RTE_FLOW_FIELD_GTP_TEID:
1420         case RTE_FLOW_FIELD_TAG:
1421                 return 32;
1422         case RTE_FLOW_FIELD_MARK:
1423                 return 24;
1424         case RTE_FLOW_FIELD_META:
1425                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_META16)
1426                         return 16;
1427                 else if (config->dv_xmeta_en == MLX5_XMETA_MODE_META32)
1428                         return 32;
1429                 else
1430                         return 0;
1431         case RTE_FLOW_FIELD_POINTER:
1432         case RTE_FLOW_FIELD_VALUE:
1433                 return 64;
1434         default:
1435                 MLX5_ASSERT(false);
1436         }
1437         return 0;
1438 }
1439
1440 static void
1441 mlx5_flow_field_id_to_modify_info
1442                 (const struct rte_flow_action_modify_data *data,
1443                  struct field_modify_info *info,
1444                  uint32_t *mask, uint32_t *value,
1445                  uint32_t width, uint32_t dst_width,
1446                  struct rte_eth_dev *dev,
1447                  const struct rte_flow_attr *attr,
1448                  struct rte_flow_error *error)
1449 {
1450         struct mlx5_priv *priv = dev->data->dev_private;
1451         struct mlx5_dev_config *config = &priv->config;
1452         uint32_t idx = 0;
1453         uint32_t off = 0;
1454         uint64_t val = 0;
1455         switch (data->field) {
1456         case RTE_FLOW_FIELD_START:
1457                 /* not supported yet */
1458                 MLX5_ASSERT(false);
1459                 break;
1460         case RTE_FLOW_FIELD_MAC_DST:
1461                 off = data->offset > 16 ? data->offset - 16 : 0;
1462                 if (mask) {
1463                         if (data->offset < 16) {
1464                                 info[idx] = (struct field_modify_info){2, 0,
1465                                                 MLX5_MODI_OUT_DMAC_15_0};
1466                                 if (width < 16) {
1467                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1468                                                                  (16 - width));
1469                                         width = 0;
1470                                 } else {
1471                                         mask[idx] = RTE_BE16(0xffff);
1472                                         width -= 16;
1473                                 }
1474                                 if (!width)
1475                                         break;
1476                                 ++idx;
1477                         }
1478                         info[idx] = (struct field_modify_info){4, 4 * idx,
1479                                                 MLX5_MODI_OUT_DMAC_47_16};
1480                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1481                                                       (32 - width)) << off);
1482                 } else {
1483                         if (data->offset < 16)
1484                                 info[idx++] = (struct field_modify_info){2, 0,
1485                                                 MLX5_MODI_OUT_DMAC_15_0};
1486                         info[idx] = (struct field_modify_info){4, off,
1487                                                 MLX5_MODI_OUT_DMAC_47_16};
1488                 }
1489                 break;
1490         case RTE_FLOW_FIELD_MAC_SRC:
1491                 off = data->offset > 16 ? data->offset - 16 : 0;
1492                 if (mask) {
1493                         if (data->offset < 16) {
1494                                 info[idx] = (struct field_modify_info){2, 0,
1495                                                 MLX5_MODI_OUT_SMAC_15_0};
1496                                 if (width < 16) {
1497                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1498                                                                  (16 - width));
1499                                         width = 0;
1500                                 } else {
1501                                         mask[idx] = RTE_BE16(0xffff);
1502                                         width -= 16;
1503                                 }
1504                                 if (!width)
1505                                         break;
1506                                 ++idx;
1507                         }
1508                         info[idx] = (struct field_modify_info){4, 4 * idx,
1509                                                 MLX5_MODI_OUT_SMAC_47_16};
1510                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1511                                                       (32 - width)) << off);
1512                 } else {
1513                         if (data->offset < 16)
1514                                 info[idx++] = (struct field_modify_info){2, 0,
1515                                                 MLX5_MODI_OUT_SMAC_15_0};
1516                         info[idx] = (struct field_modify_info){4, off,
1517                                                 MLX5_MODI_OUT_SMAC_47_16};
1518                 }
1519                 break;
1520         case RTE_FLOW_FIELD_VLAN_TYPE:
1521                 /* not supported yet */
1522                 break;
1523         case RTE_FLOW_FIELD_VLAN_ID:
1524                 info[idx] = (struct field_modify_info){2, 0,
1525                                         MLX5_MODI_OUT_FIRST_VID};
1526                 if (mask)
1527                         mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
1528                 break;
1529         case RTE_FLOW_FIELD_MAC_TYPE:
1530                 info[idx] = (struct field_modify_info){2, 0,
1531                                         MLX5_MODI_OUT_ETHERTYPE};
1532                 if (mask)
1533                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1534                 break;
1535         case RTE_FLOW_FIELD_IPV4_DSCP:
1536                 info[idx] = (struct field_modify_info){1, 0,
1537                                         MLX5_MODI_OUT_IP_DSCP};
1538                 if (mask)
1539                         mask[idx] = 0x3f >> (6 - width);
1540                 break;
1541         case RTE_FLOW_FIELD_IPV4_TTL:
1542                 info[idx] = (struct field_modify_info){1, 0,
1543                                         MLX5_MODI_OUT_IPV4_TTL};
1544                 if (mask)
1545                         mask[idx] = 0xff >> (8 - width);
1546                 break;
1547         case RTE_FLOW_FIELD_IPV4_SRC:
1548                 info[idx] = (struct field_modify_info){4, 0,
1549                                         MLX5_MODI_OUT_SIPV4};
1550                 if (mask)
1551                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1552                                                      (32 - width));
1553                 break;
1554         case RTE_FLOW_FIELD_IPV4_DST:
1555                 info[idx] = (struct field_modify_info){4, 0,
1556                                         MLX5_MODI_OUT_DIPV4};
1557                 if (mask)
1558                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1559                                                      (32 - width));
1560                 break;
1561         case RTE_FLOW_FIELD_IPV6_DSCP:
1562                 info[idx] = (struct field_modify_info){1, 0,
1563                                         MLX5_MODI_OUT_IP_DSCP};
1564                 if (mask)
1565                         mask[idx] = 0x3f >> (6 - width);
1566                 break;
1567         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1568                 info[idx] = (struct field_modify_info){1, 0,
1569                                         MLX5_MODI_OUT_IPV6_HOPLIMIT};
1570                 if (mask)
1571                         mask[idx] = 0xff >> (8 - width);
1572                 break;
1573         case RTE_FLOW_FIELD_IPV6_SRC:
1574                 if (mask) {
1575                         if (data->offset < 32) {
1576                                 info[idx] = (struct field_modify_info){4,
1577                                                 4 * idx,
1578                                                 MLX5_MODI_OUT_SIPV6_31_0};
1579                                 if (width < 32) {
1580                                         mask[idx] =
1581                                                 rte_cpu_to_be_32(0xffffffff >>
1582                                                                  (32 - width));
1583                                         width = 0;
1584                                 } else {
1585                                         mask[idx] = RTE_BE32(0xffffffff);
1586                                         width -= 32;
1587                                 }
1588                                 if (!width)
1589                                         break;
1590                                 ++idx;
1591                         }
1592                         if (data->offset < 64) {
1593                                 info[idx] = (struct field_modify_info){4,
1594                                                 4 * idx,
1595                                                 MLX5_MODI_OUT_SIPV6_63_32};
1596                                 if (width < 32) {
1597                                         mask[idx] =
1598                                                 rte_cpu_to_be_32(0xffffffff >>
1599                                                                  (32 - width));
1600                                         width = 0;
1601                                 } else {
1602                                         mask[idx] = RTE_BE32(0xffffffff);
1603                                         width -= 32;
1604                                 }
1605                                 if (!width)
1606                                         break;
1607                                 ++idx;
1608                         }
1609                         if (data->offset < 96) {
1610                                 info[idx] = (struct field_modify_info){4,
1611                                                 4 * idx,
1612                                                 MLX5_MODI_OUT_SIPV6_95_64};
1613                                 if (width < 32) {
1614                                         mask[idx] =
1615                                                 rte_cpu_to_be_32(0xffffffff >>
1616                                                                  (32 - width));
1617                                         width = 0;
1618                                 } else {
1619                                         mask[idx] = RTE_BE32(0xffffffff);
1620                                         width -= 32;
1621                                 }
1622                                 if (!width)
1623                                         break;
1624                                 ++idx;
1625                         }
1626                         info[idx] = (struct field_modify_info){4, 4 * idx,
1627                                                 MLX5_MODI_OUT_SIPV6_127_96};
1628                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1629                                                      (32 - width));
1630                 } else {
1631                         if (data->offset < 32)
1632                                 info[idx++] = (struct field_modify_info){4, 0,
1633                                                 MLX5_MODI_OUT_SIPV6_31_0};
1634                         if (data->offset < 64)
1635                                 info[idx++] = (struct field_modify_info){4, 0,
1636                                                 MLX5_MODI_OUT_SIPV6_63_32};
1637                         if (data->offset < 96)
1638                                 info[idx++] = (struct field_modify_info){4, 0,
1639                                                 MLX5_MODI_OUT_SIPV6_95_64};
1640                         if (data->offset < 128)
1641                                 info[idx++] = (struct field_modify_info){4, 0,
1642                                                 MLX5_MODI_OUT_SIPV6_127_96};
1643                 }
1644                 break;
1645         case RTE_FLOW_FIELD_IPV6_DST:
1646                 if (mask) {
1647                         if (data->offset < 32) {
1648                                 info[idx] = (struct field_modify_info){4,
1649                                                 4 * idx,
1650                                                 MLX5_MODI_OUT_DIPV6_31_0};
1651                                 if (width < 32) {
1652                                         mask[idx] =
1653                                                 rte_cpu_to_be_32(0xffffffff >>
1654                                                                  (32 - width));
1655                                         width = 0;
1656                                 } else {
1657                                         mask[idx] = RTE_BE32(0xffffffff);
1658                                         width -= 32;
1659                                 }
1660                                 if (!width)
1661                                         break;
1662                                 ++idx;
1663                         }
1664                         if (data->offset < 64) {
1665                                 info[idx] = (struct field_modify_info){4,
1666                                                 4 * idx,
1667                                                 MLX5_MODI_OUT_DIPV6_63_32};
1668                                 if (width < 32) {
1669                                         mask[idx] =
1670                                                 rte_cpu_to_be_32(0xffffffff >>
1671                                                                  (32 - width));
1672                                         width = 0;
1673                                 } else {
1674                                         mask[idx] = RTE_BE32(0xffffffff);
1675                                         width -= 32;
1676                                 }
1677                                 if (!width)
1678                                         break;
1679                                 ++idx;
1680                         }
1681                         if (data->offset < 96) {
1682                                 info[idx] = (struct field_modify_info){4,
1683                                                 4 * idx,
1684                                                 MLX5_MODI_OUT_DIPV6_95_64};
1685                                 if (width < 32) {
1686                                         mask[idx] =
1687                                                 rte_cpu_to_be_32(0xffffffff >>
1688                                                                  (32 - width));
1689                                         width = 0;
1690                                 } else {
1691                                         mask[idx] = RTE_BE32(0xffffffff);
1692                                         width -= 32;
1693                                 }
1694                                 if (!width)
1695                                         break;
1696                                 ++idx;
1697                         }
1698                         info[idx] = (struct field_modify_info){4, 4 * idx,
1699                                                 MLX5_MODI_OUT_DIPV6_127_96};
1700                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1701                                                      (32 - width));
1702                 } else {
1703                         if (data->offset < 32)
1704                                 info[idx++] = (struct field_modify_info){4, 0,
1705                                                 MLX5_MODI_OUT_DIPV6_31_0};
1706                         if (data->offset < 64)
1707                                 info[idx++] = (struct field_modify_info){4, 0,
1708                                                 MLX5_MODI_OUT_DIPV6_63_32};
1709                         if (data->offset < 96)
1710                                 info[idx++] = (struct field_modify_info){4, 0,
1711                                                 MLX5_MODI_OUT_DIPV6_95_64};
1712                         if (data->offset < 128)
1713                                 info[idx++] = (struct field_modify_info){4, 0,
1714                                                 MLX5_MODI_OUT_DIPV6_127_96};
1715                 }
1716                 break;
1717         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1718                 info[idx] = (struct field_modify_info){2, 0,
1719                                         MLX5_MODI_OUT_TCP_SPORT};
1720                 if (mask)
1721                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1722                 break;
1723         case RTE_FLOW_FIELD_TCP_PORT_DST:
1724                 info[idx] = (struct field_modify_info){2, 0,
1725                                         MLX5_MODI_OUT_TCP_DPORT};
1726                 if (mask)
1727                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1728                 break;
1729         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1730                 info[idx] = (struct field_modify_info){4, 0,
1731                                         MLX5_MODI_OUT_TCP_SEQ_NUM};
1732                 if (mask)
1733                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1734                                                      (32 - width));
1735                 break;
1736         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1737                 info[idx] = (struct field_modify_info){4, 0,
1738                                         MLX5_MODI_OUT_TCP_ACK_NUM};
1739                 if (mask)
1740                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1741                                                      (32 - width));
1742                 break;
1743         case RTE_FLOW_FIELD_TCP_FLAGS:
1744                 info[idx] = (struct field_modify_info){2, 0,
1745                                         MLX5_MODI_OUT_TCP_FLAGS};
1746                 if (mask)
1747                         mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
1748                 break;
1749         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1750                 info[idx] = (struct field_modify_info){2, 0,
1751                                         MLX5_MODI_OUT_UDP_SPORT};
1752                 if (mask)
1753                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1754                 break;
1755         case RTE_FLOW_FIELD_UDP_PORT_DST:
1756                 info[idx] = (struct field_modify_info){2, 0,
1757                                         MLX5_MODI_OUT_UDP_DPORT};
1758                 if (mask)
1759                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1760                 break;
1761         case RTE_FLOW_FIELD_VXLAN_VNI:
1762                 /* not supported yet */
1763                 break;
1764         case RTE_FLOW_FIELD_GENEVE_VNI:
1765                 /* not supported yet*/
1766                 break;
1767         case RTE_FLOW_FIELD_GTP_TEID:
1768                 info[idx] = (struct field_modify_info){4, 0,
1769                                         MLX5_MODI_GTP_TEID};
1770                 if (mask)
1771                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1772                                                      (32 - width));
1773                 break;
1774         case RTE_FLOW_FIELD_TAG:
1775                 {
1776                         int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
1777                                                    data->level, error);
1778                         if (reg < 0)
1779                                 return;
1780                         MLX5_ASSERT(reg != REG_NON);
1781                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1782                         info[idx] = (struct field_modify_info){4, 0,
1783                                                 reg_to_field[reg]};
1784                         if (mask)
1785                                 mask[idx] =
1786                                         rte_cpu_to_be_32(0xffffffff >>
1787                                                          (32 - width));
1788                 }
1789                 break;
1790         case RTE_FLOW_FIELD_MARK:
1791                 {
1792                         int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
1793                                                        0, error);
1794                         if (reg < 0)
1795                                 return;
1796                         MLX5_ASSERT(reg != REG_NON);
1797                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1798                         info[idx] = (struct field_modify_info){4, 0,
1799                                                 reg_to_field[reg]};
1800                         if (mask)
1801                                 mask[idx] =
1802                                         rte_cpu_to_be_32(0xffffffff >>
1803                                                          (32 - width));
1804                 }
1805                 break;
1806         case RTE_FLOW_FIELD_META:
1807                 {
1808                         unsigned int xmeta = config->dv_xmeta_en;
1809                         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1810                         if (reg < 0)
1811                                 return;
1812                         MLX5_ASSERT(reg != REG_NON);
1813                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1814                         if (xmeta == MLX5_XMETA_MODE_META16) {
1815                                 info[idx] = (struct field_modify_info){2, 0,
1816                                                         reg_to_field[reg]};
1817                                 if (mask)
1818                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1819                                                                 (16 - width));
1820                         } else if (xmeta == MLX5_XMETA_MODE_META32) {
1821                                 info[idx] = (struct field_modify_info){4, 0,
1822                                                         reg_to_field[reg]};
1823                                 if (mask)
1824                                         mask[idx] =
1825                                                 rte_cpu_to_be_32(0xffffffff >>
1826                                                                 (32 - width));
1827                         } else {
1828                                 MLX5_ASSERT(false);
1829                         }
1830                 }
1831                 break;
1832         case RTE_FLOW_FIELD_POINTER:
1833         case RTE_FLOW_FIELD_VALUE:
1834                 if (data->field == RTE_FLOW_FIELD_POINTER)
1835                         memcpy(&val, (void *)(uintptr_t)data->value,
1836                                sizeof(uint64_t));
1837                 else
1838                         val = data->value;
1839                 for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) {
1840                         if (mask[idx]) {
1841                                 if (dst_width == 48) {
1842                                         /*special case for MAC addresses */
1843                                         value[idx] = rte_cpu_to_be_16(val);
1844                                         val >>= 16;
1845                                         dst_width -= 16;
1846                                 } else if (dst_width > 16) {
1847                                         value[idx] = rte_cpu_to_be_32(val);
1848                                         val >>= 32;
1849                                 } else if (dst_width > 8) {
1850                                         value[idx] = rte_cpu_to_be_16(val);
1851                                         val >>= 16;
1852                                 } else {
1853                                         value[idx] = (uint8_t)val;
1854                                         val >>= 8;
1855                                 }
1856                                 if (!val)
1857                                         break;
1858                         }
1859                 }
1860                 break;
1861         default:
1862                 MLX5_ASSERT(false);
1863                 break;
1864         }
1865 }
1866
1867 /**
1868  * Convert modify_field action to DV specification.
1869  *
1870  * @param[in] dev
1871  *   Pointer to the rte_eth_dev structure.
1872  * @param[in,out] resource
1873  *   Pointer to the modify-header resource.
1874  * @param[in] action
1875  *   Pointer to action specification.
1876  * @param[in] attr
1877  *   Attributes of flow that includes this item.
1878  * @param[out] error
1879  *   Pointer to the error structure.
1880  *
1881  * @return
1882  *   0 on success, a negative errno value otherwise and rte_errno is set.
1883  */
1884 static int
1885 flow_dv_convert_action_modify_field
1886                         (struct rte_eth_dev *dev,
1887                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1888                          const struct rte_flow_action *action,
1889                          const struct rte_flow_attr *attr,
1890                          struct rte_flow_error *error)
1891 {
1892         struct mlx5_priv *priv = dev->data->dev_private;
1893         struct mlx5_dev_config *config = &priv->config;
1894         const struct rte_flow_action_modify_field *conf =
1895                 (const struct rte_flow_action_modify_field *)(action->conf);
1896         struct rte_flow_item item;
1897         struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
1898                                                                 {0, 0, 0} };
1899         struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
1900                                                                 {0, 0, 0} };
1901         uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1902         uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1903         uint32_t type;
1904         uint32_t dst_width = mlx5_flow_item_field_width(config,
1905                                                         conf->dst.field);
1906
1907         if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
1908                 conf->src.field == RTE_FLOW_FIELD_VALUE) {
1909                 type = MLX5_MODIFICATION_TYPE_SET;
1910                 /** For SET fill the destination field (field) first. */
1911                 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
1912                         value, conf->width, dst_width, dev, attr, error);
1913                 /** Then copy immediate value from source as per mask. */
1914                 mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask,
1915                         value, conf->width, dst_width, dev, attr, error);
1916                 item.spec = &value;
1917         } else {
1918                 type = MLX5_MODIFICATION_TYPE_COPY;
1919                 /** For COPY fill the destination field (dcopy) without mask. */
1920                 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
1921                         value, conf->width, dst_width, dev, attr, error);
1922                 /** Then construct the source field (field) with mask. */
1923                 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
1924                         value, conf->width, dst_width, dev, attr, error);
1925         }
1926         item.mask = &mask;
1927         return flow_dv_convert_modify_action(&item,
1928                         field, dcopy, resource, type, error);
1929 }
1930
1931 /**
1932  * Validate MARK item.
1933  *
1934  * @param[in] dev
1935  *   Pointer to the rte_eth_dev structure.
1936  * @param[in] item
1937  *   Item specification.
1938  * @param[in] attr
1939  *   Attributes of flow that includes this item.
1940  * @param[out] error
1941  *   Pointer to error structure.
1942  *
1943  * @return
1944  *   0 on success, a negative errno value otherwise and rte_errno is set.
1945  */
1946 static int
1947 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1948                            const struct rte_flow_item *item,
1949                            const struct rte_flow_attr *attr __rte_unused,
1950                            struct rte_flow_error *error)
1951 {
1952         struct mlx5_priv *priv = dev->data->dev_private;
1953         struct mlx5_dev_config *config = &priv->config;
1954         const struct rte_flow_item_mark *spec = item->spec;
1955         const struct rte_flow_item_mark *mask = item->mask;
1956         const struct rte_flow_item_mark nic_mask = {
1957                 .id = priv->sh->dv_mark_mask,
1958         };
1959         int ret;
1960
1961         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1962                 return rte_flow_error_set(error, ENOTSUP,
1963                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1964                                           "extended metadata feature"
1965                                           " isn't enabled");
1966         if (!mlx5_flow_ext_mreg_supported(dev))
1967                 return rte_flow_error_set(error, ENOTSUP,
1968                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1969                                           "extended metadata register"
1970                                           " isn't supported");
1971         if (!nic_mask.id)
1972                 return rte_flow_error_set(error, ENOTSUP,
1973                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1974                                           "extended metadata register"
1975                                           " isn't available");
1976         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1977         if (ret < 0)
1978                 return ret;
1979         if (!spec)
1980                 return rte_flow_error_set(error, EINVAL,
1981                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1982                                           item->spec,
1983                                           "data cannot be empty");
1984         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1985                 return rte_flow_error_set(error, EINVAL,
1986                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1987                                           &spec->id,
1988                                           "mark id exceeds the limit");
1989         if (!mask)
1990                 mask = &nic_mask;
1991         if (!mask->id)
1992                 return rte_flow_error_set(error, EINVAL,
1993                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1994                                         "mask cannot be zero");
1995
1996         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1997                                         (const uint8_t *)&nic_mask,
1998                                         sizeof(struct rte_flow_item_mark),
1999                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2000         if (ret < 0)
2001                 return ret;
2002         return 0;
2003 }
2004
2005 /**
2006  * Validate META item.
2007  *
2008  * @param[in] dev
2009  *   Pointer to the rte_eth_dev structure.
2010  * @param[in] item
2011  *   Item specification.
2012  * @param[in] attr
2013  *   Attributes of flow that includes this item.
2014  * @param[out] error
2015  *   Pointer to error structure.
2016  *
2017  * @return
2018  *   0 on success, a negative errno value otherwise and rte_errno is set.
2019  */
2020 static int
2021 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
2022                            const struct rte_flow_item *item,
2023                            const struct rte_flow_attr *attr,
2024                            struct rte_flow_error *error)
2025 {
2026         struct mlx5_priv *priv = dev->data->dev_private;
2027         struct mlx5_dev_config *config = &priv->config;
2028         const struct rte_flow_item_meta *spec = item->spec;
2029         const struct rte_flow_item_meta *mask = item->mask;
2030         struct rte_flow_item_meta nic_mask = {
2031                 .data = UINT32_MAX
2032         };
2033         int reg;
2034         int ret;
2035
2036         if (!spec)
2037                 return rte_flow_error_set(error, EINVAL,
2038                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2039                                           item->spec,
2040                                           "data cannot be empty");
2041         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2042                 if (!mlx5_flow_ext_mreg_supported(dev))
2043                         return rte_flow_error_set(error, ENOTSUP,
2044                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2045                                           "extended metadata register"
2046                                           " isn't supported");
2047                 reg = flow_dv_get_metadata_reg(dev, attr, error);
2048                 if (reg < 0)
2049                         return reg;
2050                 if (reg == REG_NON)
2051                         return rte_flow_error_set(error, ENOTSUP,
2052                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2053                                         "unavalable extended metadata register");
2054                 if (reg == REG_B)
2055                         return rte_flow_error_set(error, ENOTSUP,
2056                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2057                                           "match on reg_b "
2058                                           "isn't supported");
2059                 if (reg != REG_A)
2060                         nic_mask.data = priv->sh->dv_meta_mask;
2061         } else {
2062                 if (attr->transfer)
2063                         return rte_flow_error_set(error, ENOTSUP,
2064                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2065                                         "extended metadata feature "
2066                                         "should be enabled when "
2067                                         "meta item is requested "
2068                                         "with e-switch mode ");
2069                 if (attr->ingress)
2070                         return rte_flow_error_set(error, ENOTSUP,
2071                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2072                                         "match on metadata for ingress "
2073                                         "is not supported in legacy "
2074                                         "metadata mode");
2075         }
2076         if (!mask)
2077                 mask = &rte_flow_item_meta_mask;
2078         if (!mask->data)
2079                 return rte_flow_error_set(error, EINVAL,
2080                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2081                                         "mask cannot be zero");
2082
2083         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2084                                         (const uint8_t *)&nic_mask,
2085                                         sizeof(struct rte_flow_item_meta),
2086                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2087         return ret;
2088 }
2089
2090 /**
2091  * Validate TAG item.
2092  *
2093  * @param[in] dev
2094  *   Pointer to the rte_eth_dev structure.
2095  * @param[in] item
2096  *   Item specification.
2097  * @param[in] attr
2098  *   Attributes of flow that includes this item.
2099  * @param[out] error
2100  *   Pointer to error structure.
2101  *
2102  * @return
2103  *   0 on success, a negative errno value otherwise and rte_errno is set.
2104  */
2105 static int
2106 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
2107                           const struct rte_flow_item *item,
2108                           const struct rte_flow_attr *attr __rte_unused,
2109                           struct rte_flow_error *error)
2110 {
2111         const struct rte_flow_item_tag *spec = item->spec;
2112         const struct rte_flow_item_tag *mask = item->mask;
2113         const struct rte_flow_item_tag nic_mask = {
2114                 .data = RTE_BE32(UINT32_MAX),
2115                 .index = 0xff,
2116         };
2117         int ret;
2118
2119         if (!mlx5_flow_ext_mreg_supported(dev))
2120                 return rte_flow_error_set(error, ENOTSUP,
2121                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2122                                           "extensive metadata register"
2123                                           " isn't supported");
2124         if (!spec)
2125                 return rte_flow_error_set(error, EINVAL,
2126                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2127                                           item->spec,
2128                                           "data cannot be empty");
2129         if (!mask)
2130                 mask = &rte_flow_item_tag_mask;
2131         if (!mask->data)
2132                 return rte_flow_error_set(error, EINVAL,
2133                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2134                                         "mask cannot be zero");
2135
2136         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2137                                         (const uint8_t *)&nic_mask,
2138                                         sizeof(struct rte_flow_item_tag),
2139                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2140         if (ret < 0)
2141                 return ret;
2142         if (mask->index != 0xff)
2143                 return rte_flow_error_set(error, EINVAL,
2144                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2145                                           "partial mask for tag index"
2146                                           " is not supported");
2147         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
2148         if (ret < 0)
2149                 return ret;
2150         MLX5_ASSERT(ret != REG_NON);
2151         return 0;
2152 }
2153
2154 /**
2155  * Validate vport item.
2156  *
2157  * @param[in] dev
2158  *   Pointer to the rte_eth_dev structure.
2159  * @param[in] item
2160  *   Item specification.
2161  * @param[in] attr
2162  *   Attributes of flow that includes this item.
2163  * @param[in] item_flags
2164  *   Bit-fields that holds the items detected until now.
2165  * @param[out] error
2166  *   Pointer to error structure.
2167  *
2168  * @return
2169  *   0 on success, a negative errno value otherwise and rte_errno is set.
2170  */
2171 static int
2172 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
2173                               const struct rte_flow_item *item,
2174                               const struct rte_flow_attr *attr,
2175                               uint64_t item_flags,
2176                               struct rte_flow_error *error)
2177 {
2178         const struct rte_flow_item_port_id *spec = item->spec;
2179         const struct rte_flow_item_port_id *mask = item->mask;
2180         const struct rte_flow_item_port_id switch_mask = {
2181                         .id = 0xffffffff,
2182         };
2183         struct mlx5_priv *esw_priv;
2184         struct mlx5_priv *dev_priv;
2185         int ret;
2186
2187         if (!attr->transfer)
2188                 return rte_flow_error_set(error, EINVAL,
2189                                           RTE_FLOW_ERROR_TYPE_ITEM,
2190                                           NULL,
2191                                           "match on port id is valid only"
2192                                           " when transfer flag is enabled");
2193         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
2194                 return rte_flow_error_set(error, ENOTSUP,
2195                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2196                                           "multiple source ports are not"
2197                                           " supported");
2198         if (!mask)
2199                 mask = &switch_mask;
2200         if (mask->id != 0xffffffff)
2201                 return rte_flow_error_set(error, ENOTSUP,
2202                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2203                                            mask,
2204                                            "no support for partial mask on"
2205                                            " \"id\" field");
2206         ret = mlx5_flow_item_acceptable
2207                                 (item, (const uint8_t *)mask,
2208                                  (const uint8_t *)&rte_flow_item_port_id_mask,
2209                                  sizeof(struct rte_flow_item_port_id),
2210                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2211         if (ret)
2212                 return ret;
2213         if (!spec)
2214                 return 0;
2215         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
2216         if (!esw_priv)
2217                 return rte_flow_error_set(error, rte_errno,
2218                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2219                                           "failed to obtain E-Switch info for"
2220                                           " port");
2221         dev_priv = mlx5_dev_to_eswitch_info(dev);
2222         if (!dev_priv)
2223                 return rte_flow_error_set(error, rte_errno,
2224                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2225                                           NULL,
2226                                           "failed to obtain E-Switch info");
2227         if (esw_priv->domain_id != dev_priv->domain_id)
2228                 return rte_flow_error_set(error, EINVAL,
2229                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2230                                           "cannot match on a port from a"
2231                                           " different E-Switch");
2232         return 0;
2233 }
2234
2235 /**
2236  * Validate VLAN item.
2237  *
2238  * @param[in] item
2239  *   Item specification.
2240  * @param[in] item_flags
2241  *   Bit-fields that holds the items detected until now.
2242  * @param[in] dev
2243  *   Ethernet device flow is being created on.
2244  * @param[out] error
2245  *   Pointer to error structure.
2246  *
2247  * @return
2248  *   0 on success, a negative errno value otherwise and rte_errno is set.
2249  */
2250 static int
2251 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
2252                            uint64_t item_flags,
2253                            struct rte_eth_dev *dev,
2254                            struct rte_flow_error *error)
2255 {
2256         const struct rte_flow_item_vlan *mask = item->mask;
2257         const struct rte_flow_item_vlan nic_mask = {
2258                 .tci = RTE_BE16(UINT16_MAX),
2259                 .inner_type = RTE_BE16(UINT16_MAX),
2260                 .has_more_vlan = 1,
2261         };
2262         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2263         int ret;
2264         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2265                                         MLX5_FLOW_LAYER_INNER_L4) :
2266                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2267                                         MLX5_FLOW_LAYER_OUTER_L4);
2268         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2269                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2270
2271         if (item_flags & vlanm)
2272                 return rte_flow_error_set(error, EINVAL,
2273                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2274                                           "multiple VLAN layers not supported");
2275         else if ((item_flags & l34m) != 0)
2276                 return rte_flow_error_set(error, EINVAL,
2277                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2278                                           "VLAN cannot follow L3/L4 layer");
2279         if (!mask)
2280                 mask = &rte_flow_item_vlan_mask;
2281         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2282                                         (const uint8_t *)&nic_mask,
2283                                         sizeof(struct rte_flow_item_vlan),
2284                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2285         if (ret)
2286                 return ret;
2287         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2288                 struct mlx5_priv *priv = dev->data->dev_private;
2289
2290                 if (priv->vmwa_context) {
2291                         /*
2292                          * Non-NULL context means we have a virtual machine
2293                          * and SR-IOV enabled, we have to create VLAN interface
2294                          * to make hypervisor to setup E-Switch vport
2295                          * context correctly. We avoid creating the multiple
2296                          * VLAN interfaces, so we cannot support VLAN tag mask.
2297                          */
2298                         return rte_flow_error_set(error, EINVAL,
2299                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2300                                                   item,
2301                                                   "VLAN tag mask is not"
2302                                                   " supported in virtual"
2303                                                   " environment");
2304                 }
2305         }
2306         return 0;
2307 }
2308
2309 /*
2310  * GTP flags are contained in 1 byte of the format:
2311  * -------------------------------------------
2312  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
2313  * |-----------------------------------------|
2314  * | value | Version | PT | Res | E | S | PN |
2315  * -------------------------------------------
2316  *
2317  * Matching is supported only for GTP flags E, S, PN.
2318  */
2319 #define MLX5_GTP_FLAGS_MASK     0x07
2320
2321 /**
2322  * Validate GTP item.
2323  *
2324  * @param[in] dev
2325  *   Pointer to the rte_eth_dev structure.
2326  * @param[in] item
2327  *   Item specification.
2328  * @param[in] item_flags
2329  *   Bit-fields that holds the items detected until now.
2330  * @param[out] error
2331  *   Pointer to error structure.
2332  *
2333  * @return
2334  *   0 on success, a negative errno value otherwise and rte_errno is set.
2335  */
2336 static int
2337 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
2338                           const struct rte_flow_item *item,
2339                           uint64_t item_flags,
2340                           struct rte_flow_error *error)
2341 {
2342         struct mlx5_priv *priv = dev->data->dev_private;
2343         const struct rte_flow_item_gtp *spec = item->spec;
2344         const struct rte_flow_item_gtp *mask = item->mask;
2345         const struct rte_flow_item_gtp nic_mask = {
2346                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
2347                 .msg_type = 0xff,
2348                 .teid = RTE_BE32(0xffffffff),
2349         };
2350
2351         if (!priv->config.hca_attr.tunnel_stateless_gtp)
2352                 return rte_flow_error_set(error, ENOTSUP,
2353                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2354                                           "GTP support is not enabled");
2355         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2356                 return rte_flow_error_set(error, ENOTSUP,
2357                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2358                                           "multiple tunnel layers not"
2359                                           " supported");
2360         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2361                 return rte_flow_error_set(error, EINVAL,
2362                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2363                                           "no outer UDP layer found");
2364         if (!mask)
2365                 mask = &rte_flow_item_gtp_mask;
2366         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
2367                 return rte_flow_error_set(error, ENOTSUP,
2368                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2369                                           "Match is supported for GTP"
2370                                           " flags only");
2371         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2372                                          (const uint8_t *)&nic_mask,
2373                                          sizeof(struct rte_flow_item_gtp),
2374                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2375 }
2376
2377 /**
2378  * Validate GTP PSC item.
2379  *
2380  * @param[in] item
2381  *   Item specification.
2382  * @param[in] last_item
2383  *   Previous validated item in the pattern items.
2384  * @param[in] gtp_item
2385  *   Previous GTP item specification.
2386  * @param[in] attr
2387  *   Pointer to flow attributes.
2388  * @param[out] error
2389  *   Pointer to error structure.
2390  *
2391  * @return
2392  *   0 on success, a negative errno value otherwise and rte_errno is set.
2393  */
2394 static int
2395 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
2396                               uint64_t last_item,
2397                               const struct rte_flow_item *gtp_item,
2398                               const struct rte_flow_attr *attr,
2399                               struct rte_flow_error *error)
2400 {
2401         const struct rte_flow_item_gtp *gtp_spec;
2402         const struct rte_flow_item_gtp *gtp_mask;
2403         const struct rte_flow_item_gtp_psc *spec;
2404         const struct rte_flow_item_gtp_psc *mask;
2405         const struct rte_flow_item_gtp_psc nic_mask = {
2406                 .pdu_type = 0xFF,
2407                 .qfi = 0xFF,
2408         };
2409
2410         if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
2411                 return rte_flow_error_set
2412                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2413                          "GTP PSC item must be preceded with GTP item");
2414         gtp_spec = gtp_item->spec;
2415         gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
2416         /* GTP spec and E flag is requested to match zero. */
2417         if (gtp_spec &&
2418                 (gtp_mask->v_pt_rsv_flags &
2419                 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
2420                 return rte_flow_error_set
2421                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2422                          "GTP E flag must be 1 to match GTP PSC");
2423         /* Check the flow is not created in group zero. */
2424         if (!attr->transfer && !attr->group)
2425                 return rte_flow_error_set
2426                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2427                          "GTP PSC is not supported for group 0");
2428         /* GTP spec is here and E flag is requested to match zero. */
2429         if (!item->spec)
2430                 return 0;
2431         spec = item->spec;
2432         mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
2433         if (spec->pdu_type > MLX5_GTP_EXT_MAX_PDU_TYPE)
2434                 return rte_flow_error_set
2435                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2436                          "PDU type should be smaller than 16");
2437         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2438                                          (const uint8_t *)&nic_mask,
2439                                          sizeof(struct rte_flow_item_gtp_psc),
2440                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2441 }
2442
2443 /**
2444  * Validate IPV4 item.
2445  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
2446  * add specific validation of fragment_offset field,
2447  *
2448  * @param[in] item
2449  *   Item specification.
2450  * @param[in] item_flags
2451  *   Bit-fields that holds the items detected until now.
2452  * @param[out] error
2453  *   Pointer to error structure.
2454  *
2455  * @return
2456  *   0 on success, a negative errno value otherwise and rte_errno is set.
2457  */
2458 static int
2459 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
2460                            uint64_t item_flags,
2461                            uint64_t last_item,
2462                            uint16_t ether_type,
2463                            struct rte_flow_error *error)
2464 {
2465         int ret;
2466         const struct rte_flow_item_ipv4 *spec = item->spec;
2467         const struct rte_flow_item_ipv4 *last = item->last;
2468         const struct rte_flow_item_ipv4 *mask = item->mask;
2469         rte_be16_t fragment_offset_spec = 0;
2470         rte_be16_t fragment_offset_last = 0;
2471         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
2472                 .hdr = {
2473                         .src_addr = RTE_BE32(0xffffffff),
2474                         .dst_addr = RTE_BE32(0xffffffff),
2475                         .type_of_service = 0xff,
2476                         .fragment_offset = RTE_BE16(0xffff),
2477                         .next_proto_id = 0xff,
2478                         .time_to_live = 0xff,
2479                 },
2480         };
2481
2482         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
2483                                            ether_type, &nic_ipv4_mask,
2484                                            MLX5_ITEM_RANGE_ACCEPTED, error);
2485         if (ret < 0)
2486                 return ret;
2487         if (spec && mask)
2488                 fragment_offset_spec = spec->hdr.fragment_offset &
2489                                        mask->hdr.fragment_offset;
2490         if (!fragment_offset_spec)
2491                 return 0;
2492         /*
2493          * spec and mask are valid, enforce using full mask to make sure the
2494          * complete value is used correctly.
2495          */
2496         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2497                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2498                 return rte_flow_error_set(error, EINVAL,
2499                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2500                                           item, "must use full mask for"
2501                                           " fragment_offset");
2502         /*
2503          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
2504          * indicating this is 1st fragment of fragmented packet.
2505          * This is not yet supported in MLX5, return appropriate error message.
2506          */
2507         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
2508                 return rte_flow_error_set(error, ENOTSUP,
2509                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2510                                           "match on first fragment not "
2511                                           "supported");
2512         if (fragment_offset_spec && !last)
2513                 return rte_flow_error_set(error, ENOTSUP,
2514                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2515                                           "specified value not supported");
2516         /* spec and last are valid, validate the specified range. */
2517         fragment_offset_last = last->hdr.fragment_offset &
2518                                mask->hdr.fragment_offset;
2519         /*
2520          * Match on fragment_offset spec 0x2001 and last 0x3fff
2521          * means MF is 1 and frag-offset is > 0.
2522          * This packet is fragment 2nd and onward, excluding last.
2523          * This is not yet supported in MLX5, return appropriate
2524          * error message.
2525          */
2526         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
2527             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2528                 return rte_flow_error_set(error, ENOTSUP,
2529                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2530                                           last, "match on following "
2531                                           "fragments not supported");
2532         /*
2533          * Match on fragment_offset spec 0x0001 and last 0x1fff
2534          * means MF is 0 and frag-offset is > 0.
2535          * This packet is last fragment of fragmented packet.
2536          * This is not yet supported in MLX5, return appropriate
2537          * error message.
2538          */
2539         if (fragment_offset_spec == RTE_BE16(1) &&
2540             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
2541                 return rte_flow_error_set(error, ENOTSUP,
2542                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2543                                           last, "match on last "
2544                                           "fragment not supported");
2545         /*
2546          * Match on fragment_offset spec 0x0001 and last 0x3fff
2547          * means MF and/or frag-offset is not 0.
2548          * This is a fragmented packet.
2549          * Other range values are invalid and rejected.
2550          */
2551         if (!(fragment_offset_spec == RTE_BE16(1) &&
2552               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
2553                 return rte_flow_error_set(error, ENOTSUP,
2554                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2555                                           "specified range not supported");
2556         return 0;
2557 }
2558
2559 /**
2560  * Validate IPV6 fragment extension item.
2561  *
2562  * @param[in] item
2563  *   Item specification.
2564  * @param[in] item_flags
2565  *   Bit-fields that holds the items detected until now.
2566  * @param[out] error
2567  *   Pointer to error structure.
2568  *
2569  * @return
2570  *   0 on success, a negative errno value otherwise and rte_errno is set.
2571  */
2572 static int
2573 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
2574                                     uint64_t item_flags,
2575                                     struct rte_flow_error *error)
2576 {
2577         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
2578         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
2579         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
2580         rte_be16_t frag_data_spec = 0;
2581         rte_be16_t frag_data_last = 0;
2582         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2583         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2584                                       MLX5_FLOW_LAYER_OUTER_L4;
2585         int ret = 0;
2586         struct rte_flow_item_ipv6_frag_ext nic_mask = {
2587                 .hdr = {
2588                         .next_header = 0xff,
2589                         .frag_data = RTE_BE16(0xffff),
2590                 },
2591         };
2592
2593         if (item_flags & l4m)
2594                 return rte_flow_error_set(error, EINVAL,
2595                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2596                                           "ipv6 fragment extension item cannot "
2597                                           "follow L4 item.");
2598         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
2599             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2600                 return rte_flow_error_set(error, EINVAL,
2601                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2602                                           "ipv6 fragment extension item must "
2603                                           "follow ipv6 item");
2604         if (spec && mask)
2605                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
2606         if (!frag_data_spec)
2607                 return 0;
2608         /*
2609          * spec and mask are valid, enforce using full mask to make sure the
2610          * complete value is used correctly.
2611          */
2612         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
2613                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2614                 return rte_flow_error_set(error, EINVAL,
2615                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2616                                           item, "must use full mask for"
2617                                           " frag_data");
2618         /*
2619          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
2620          * This is 1st fragment of fragmented packet.
2621          */
2622         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
2623                 return rte_flow_error_set(error, ENOTSUP,
2624                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2625                                           "match on first fragment not "
2626                                           "supported");
2627         if (frag_data_spec && !last)
2628                 return rte_flow_error_set(error, EINVAL,
2629                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2630                                           "specified value not supported");
2631         ret = mlx5_flow_item_acceptable
2632                                 (item, (const uint8_t *)mask,
2633                                  (const uint8_t *)&nic_mask,
2634                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
2635                                  MLX5_ITEM_RANGE_ACCEPTED, error);
2636         if (ret)
2637                 return ret;
2638         /* spec and last are valid, validate the specified range. */
2639         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
2640         /*
2641          * Match on frag_data spec 0x0009 and last 0xfff9
2642          * means M is 1 and frag-offset is > 0.
2643          * This packet is fragment 2nd and onward, excluding last.
2644          * This is not yet supported in MLX5, return appropriate
2645          * error message.
2646          */
2647         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2648                                        RTE_IPV6_EHDR_MF_MASK) &&
2649             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2650                 return rte_flow_error_set(error, ENOTSUP,
2651                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2652                                           last, "match on following "
2653                                           "fragments not supported");
2654         /*
2655          * Match on frag_data spec 0x0008 and last 0xfff8
2656          * means M is 0 and frag-offset is > 0.
2657          * This packet is last fragment of fragmented packet.
2658          * This is not yet supported in MLX5, return appropriate
2659          * error message.
2660          */
2661         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2662             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2663                 return rte_flow_error_set(error, ENOTSUP,
2664                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2665                                           last, "match on last "
2666                                           "fragment not supported");
2667         /* Other range values are invalid and rejected. */
2668         return rte_flow_error_set(error, EINVAL,
2669                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2670                                   "specified range not supported");
2671 }
2672
2673 /*
2674  * Validate ASO CT item.
2675  *
2676  * @param[in] dev
2677  *   Pointer to the rte_eth_dev structure.
2678  * @param[in] item
2679  *   Item specification.
2680  * @param[in] item_flags
2681  *   Pointer to bit-fields that holds the items detected until now.
2682  * @param[out] error
2683  *   Pointer to error structure.
2684  *
2685  * @return
2686  *   0 on success, a negative errno value otherwise and rte_errno is set.
2687  */
2688 static int
2689 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
2690                              const struct rte_flow_item *item,
2691                              uint64_t *item_flags,
2692                              struct rte_flow_error *error)
2693 {
2694         const struct rte_flow_item_conntrack *spec = item->spec;
2695         const struct rte_flow_item_conntrack *mask = item->mask;
2696         RTE_SET_USED(dev);
2697         uint32_t flags;
2698
2699         if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
2700                 return rte_flow_error_set(error, EINVAL,
2701                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2702                                           "Only one CT is supported");
2703         if (!mask)
2704                 mask = &rte_flow_item_conntrack_mask;
2705         flags = spec->flags & mask->flags;
2706         if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
2707             ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
2708              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
2709              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
2710                 return rte_flow_error_set(error, EINVAL,
2711                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2712                                           "Conflict status bits");
2713         /* State change also needs to be considered. */
2714         *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
2715         return 0;
2716 }
2717
2718 /**
2719  * Validate the pop VLAN action.
2720  *
2721  * @param[in] dev
2722  *   Pointer to the rte_eth_dev structure.
2723  * @param[in] action_flags
2724  *   Holds the actions detected until now.
2725  * @param[in] action
2726  *   Pointer to the pop vlan action.
2727  * @param[in] item_flags
2728  *   The items found in this flow rule.
2729  * @param[in] attr
2730  *   Pointer to flow attributes.
2731  * @param[out] error
2732  *   Pointer to error structure.
2733  *
2734  * @return
2735  *   0 on success, a negative errno value otherwise and rte_errno is set.
2736  */
2737 static int
2738 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2739                                  uint64_t action_flags,
2740                                  const struct rte_flow_action *action,
2741                                  uint64_t item_flags,
2742                                  const struct rte_flow_attr *attr,
2743                                  struct rte_flow_error *error)
2744 {
2745         const struct mlx5_priv *priv = dev->data->dev_private;
2746
2747         (void)action;
2748         (void)attr;
2749         if (!priv->sh->pop_vlan_action)
2750                 return rte_flow_error_set(error, ENOTSUP,
2751                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2752                                           NULL,
2753                                           "pop vlan action is not supported");
2754         if (attr->egress)
2755                 return rte_flow_error_set(error, ENOTSUP,
2756                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2757                                           NULL,
2758                                           "pop vlan action not supported for "
2759                                           "egress");
2760         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2761                 return rte_flow_error_set(error, ENOTSUP,
2762                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2763                                           "no support for multiple VLAN "
2764                                           "actions");
2765         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2766         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2767             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2768                 return rte_flow_error_set(error, ENOTSUP,
2769                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2770                                           NULL,
2771                                           "cannot pop vlan after decap without "
2772                                           "match on inner vlan in the flow");
2773         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2774         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2775             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2776                 return rte_flow_error_set(error, ENOTSUP,
2777                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2778                                           NULL,
2779                                           "cannot pop vlan without a "
2780                                           "match on (outer) vlan in the flow");
2781         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2782                 return rte_flow_error_set(error, EINVAL,
2783                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2784                                           "wrong action order, port_id should "
2785                                           "be after pop VLAN action");
2786         if (!attr->transfer && priv->representor)
2787                 return rte_flow_error_set(error, ENOTSUP,
2788                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2789                                           "pop vlan action for VF representor "
2790                                           "not supported on NIC table");
2791         return 0;
2792 }
2793
2794 /**
2795  * Get VLAN default info from vlan match info.
2796  *
2797  * @param[in] items
2798  *   the list of item specifications.
2799  * @param[out] vlan
2800  *   pointer VLAN info to fill to.
2801  *
2802  * @return
2803  *   0 on success, a negative errno value otherwise and rte_errno is set.
2804  */
2805 static void
2806 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2807                                   struct rte_vlan_hdr *vlan)
2808 {
2809         const struct rte_flow_item_vlan nic_mask = {
2810                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2811                                 MLX5DV_FLOW_VLAN_VID_MASK),
2812                 .inner_type = RTE_BE16(0xffff),
2813         };
2814
2815         if (items == NULL)
2816                 return;
2817         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2818                 int type = items->type;
2819
2820                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2821                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2822                         break;
2823         }
2824         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2825                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2826                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2827
2828                 /* If VLAN item in pattern doesn't contain data, return here. */
2829                 if (!vlan_v)
2830                         return;
2831                 if (!vlan_m)
2832                         vlan_m = &nic_mask;
2833                 /* Only full match values are accepted */
2834                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2835                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2836                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2837                         vlan->vlan_tci |=
2838                                 rte_be_to_cpu_16(vlan_v->tci &
2839                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2840                 }
2841                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2842                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2843                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2844                         vlan->vlan_tci |=
2845                                 rte_be_to_cpu_16(vlan_v->tci &
2846                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2847                 }
2848                 if (vlan_m->inner_type == nic_mask.inner_type)
2849                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2850                                                            vlan_m->inner_type);
2851         }
2852 }
2853
2854 /**
2855  * Validate the push VLAN action.
2856  *
2857  * @param[in] dev
2858  *   Pointer to the rte_eth_dev structure.
2859  * @param[in] action_flags
2860  *   Holds the actions detected until now.
2861  * @param[in] item_flags
2862  *   The items found in this flow rule.
2863  * @param[in] action
2864  *   Pointer to the action structure.
2865  * @param[in] attr
2866  *   Pointer to flow attributes
2867  * @param[out] error
2868  *   Pointer to error structure.
2869  *
2870  * @return
2871  *   0 on success, a negative errno value otherwise and rte_errno is set.
2872  */
2873 static int
2874 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2875                                   uint64_t action_flags,
2876                                   const struct rte_flow_item_vlan *vlan_m,
2877                                   const struct rte_flow_action *action,
2878                                   const struct rte_flow_attr *attr,
2879                                   struct rte_flow_error *error)
2880 {
2881         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2882         const struct mlx5_priv *priv = dev->data->dev_private;
2883
2884         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2885             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2886                 return rte_flow_error_set(error, EINVAL,
2887                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2888                                           "invalid vlan ethertype");
2889         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2890                 return rte_flow_error_set(error, EINVAL,
2891                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2892                                           "wrong action order, port_id should "
2893                                           "be after push VLAN");
2894         if (!attr->transfer && priv->representor)
2895                 return rte_flow_error_set(error, ENOTSUP,
2896                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2897                                           "push vlan action for VF representor "
2898                                           "not supported on NIC table");
2899         if (vlan_m &&
2900             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2901             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2902                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2903             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2904             !(mlx5_flow_find_action
2905                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2906                 return rte_flow_error_set(error, EINVAL,
2907                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2908                                           "not full match mask on VLAN PCP and "
2909                                           "there is no of_set_vlan_pcp action, "
2910                                           "push VLAN action cannot figure out "
2911                                           "PCP value");
2912         if (vlan_m &&
2913             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2914             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2915                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2916             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2917             !(mlx5_flow_find_action
2918                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2919                 return rte_flow_error_set(error, EINVAL,
2920                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2921                                           "not full match mask on VLAN VID and "
2922                                           "there is no of_set_vlan_vid action, "
2923                                           "push VLAN action cannot figure out "
2924                                           "VID value");
2925         (void)attr;
2926         return 0;
2927 }
2928
2929 /**
2930  * Validate the set VLAN PCP.
2931  *
2932  * @param[in] action_flags
2933  *   Holds the actions detected until now.
2934  * @param[in] actions
2935  *   Pointer to the list of actions remaining in the flow rule.
2936  * @param[out] error
2937  *   Pointer to error structure.
2938  *
2939  * @return
2940  *   0 on success, a negative errno value otherwise and rte_errno is set.
2941  */
2942 static int
2943 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2944                                      const struct rte_flow_action actions[],
2945                                      struct rte_flow_error *error)
2946 {
2947         const struct rte_flow_action *action = actions;
2948         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2949
2950         if (conf->vlan_pcp > 7)
2951                 return rte_flow_error_set(error, EINVAL,
2952                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2953                                           "VLAN PCP value is too big");
2954         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2955                 return rte_flow_error_set(error, ENOTSUP,
2956                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2957                                           "set VLAN PCP action must follow "
2958                                           "the push VLAN action");
2959         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2960                 return rte_flow_error_set(error, ENOTSUP,
2961                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2962                                           "Multiple VLAN PCP modification are "
2963                                           "not supported");
2964         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2965                 return rte_flow_error_set(error, EINVAL,
2966                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2967                                           "wrong action order, port_id should "
2968                                           "be after set VLAN PCP");
2969         return 0;
2970 }
2971
2972 /**
2973  * Validate the set VLAN VID.
2974  *
2975  * @param[in] item_flags
2976  *   Holds the items detected in this rule.
2977  * @param[in] action_flags
2978  *   Holds the actions detected until now.
2979  * @param[in] actions
2980  *   Pointer to the list of actions remaining in the flow rule.
2981  * @param[out] error
2982  *   Pointer to error structure.
2983  *
2984  * @return
2985  *   0 on success, a negative errno value otherwise and rte_errno is set.
2986  */
2987 static int
2988 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2989                                      uint64_t action_flags,
2990                                      const struct rte_flow_action actions[],
2991                                      struct rte_flow_error *error)
2992 {
2993         const struct rte_flow_action *action = actions;
2994         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2995
2996         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2997                 return rte_flow_error_set(error, EINVAL,
2998                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2999                                           "VLAN VID value is too big");
3000         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
3001             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
3002                 return rte_flow_error_set(error, ENOTSUP,
3003                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3004                                           "set VLAN VID action must follow push"
3005                                           " VLAN action or match on VLAN item");
3006         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
3007                 return rte_flow_error_set(error, ENOTSUP,
3008                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3009                                           "Multiple VLAN VID modifications are "
3010                                           "not supported");
3011         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3012                 return rte_flow_error_set(error, EINVAL,
3013                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3014                                           "wrong action order, port_id should "
3015                                           "be after set VLAN VID");
3016         return 0;
3017 }
3018
3019 /*
3020  * Validate the FLAG action.
3021  *
3022  * @param[in] dev
3023  *   Pointer to the rte_eth_dev structure.
3024  * @param[in] action_flags
3025  *   Holds the actions detected until now.
3026  * @param[in] attr
3027  *   Pointer to flow attributes
3028  * @param[out] error
3029  *   Pointer to error structure.
3030  *
3031  * @return
3032  *   0 on success, a negative errno value otherwise and rte_errno is set.
3033  */
3034 static int
3035 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
3036                              uint64_t action_flags,
3037                              const struct rte_flow_attr *attr,
3038                              struct rte_flow_error *error)
3039 {
3040         struct mlx5_priv *priv = dev->data->dev_private;
3041         struct mlx5_dev_config *config = &priv->config;
3042         int ret;
3043
3044         /* Fall back if no extended metadata register support. */
3045         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3046                 return mlx5_flow_validate_action_flag(action_flags, attr,
3047                                                       error);
3048         /* Extensive metadata mode requires registers. */
3049         if (!mlx5_flow_ext_mreg_supported(dev))
3050                 return rte_flow_error_set(error, ENOTSUP,
3051                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3052                                           "no metadata registers "
3053                                           "to support flag action");
3054         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
3055                 return rte_flow_error_set(error, ENOTSUP,
3056                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3057                                           "extended metadata register"
3058                                           " isn't available");
3059         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3060         if (ret < 0)
3061                 return ret;
3062         MLX5_ASSERT(ret > 0);
3063         if (action_flags & MLX5_FLOW_ACTION_MARK)
3064                 return rte_flow_error_set(error, EINVAL,
3065                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3066                                           "can't mark and flag in same flow");
3067         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3068                 return rte_flow_error_set(error, EINVAL,
3069                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3070                                           "can't have 2 flag"
3071                                           " actions in same flow");
3072         return 0;
3073 }
3074
3075 /**
3076  * Validate MARK action.
3077  *
3078  * @param[in] dev
3079  *   Pointer to the rte_eth_dev structure.
3080  * @param[in] action
3081  *   Pointer to action.
3082  * @param[in] action_flags
3083  *   Holds the actions detected until now.
3084  * @param[in] attr
3085  *   Pointer to flow attributes
3086  * @param[out] error
3087  *   Pointer to error structure.
3088  *
3089  * @return
3090  *   0 on success, a negative errno value otherwise and rte_errno is set.
3091  */
3092 static int
3093 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
3094                              const struct rte_flow_action *action,
3095                              uint64_t action_flags,
3096                              const struct rte_flow_attr *attr,
3097                              struct rte_flow_error *error)
3098 {
3099         struct mlx5_priv *priv = dev->data->dev_private;
3100         struct mlx5_dev_config *config = &priv->config;
3101         const struct rte_flow_action_mark *mark = action->conf;
3102         int ret;
3103
3104         if (is_tunnel_offload_active(dev))
3105                 return rte_flow_error_set(error, ENOTSUP,
3106                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3107                                           "no mark action "
3108                                           "if tunnel offload active");
3109         /* Fall back if no extended metadata register support. */
3110         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3111                 return mlx5_flow_validate_action_mark(action, action_flags,
3112                                                       attr, error);
3113         /* Extensive metadata mode requires registers. */
3114         if (!mlx5_flow_ext_mreg_supported(dev))
3115                 return rte_flow_error_set(error, ENOTSUP,
3116                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3117                                           "no metadata registers "
3118                                           "to support mark action");
3119         if (!priv->sh->dv_mark_mask)
3120                 return rte_flow_error_set(error, ENOTSUP,
3121                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3122                                           "extended metadata register"
3123                                           " isn't available");
3124         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3125         if (ret < 0)
3126                 return ret;
3127         MLX5_ASSERT(ret > 0);
3128         if (!mark)
3129                 return rte_flow_error_set(error, EINVAL,
3130                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3131                                           "configuration cannot be null");
3132         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
3133                 return rte_flow_error_set(error, EINVAL,
3134                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3135                                           &mark->id,
3136                                           "mark id exceeds the limit");
3137         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3138                 return rte_flow_error_set(error, EINVAL,
3139                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3140                                           "can't flag and mark in same flow");
3141         if (action_flags & MLX5_FLOW_ACTION_MARK)
3142                 return rte_flow_error_set(error, EINVAL,
3143                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3144                                           "can't have 2 mark actions in same"
3145                                           " flow");
3146         return 0;
3147 }
3148
3149 /**
3150  * Validate SET_META action.
3151  *
3152  * @param[in] dev
3153  *   Pointer to the rte_eth_dev structure.
3154  * @param[in] action
3155  *   Pointer to the action structure.
3156  * @param[in] action_flags
3157  *   Holds the actions detected until now.
3158  * @param[in] attr
3159  *   Pointer to flow attributes
3160  * @param[out] error
3161  *   Pointer to error structure.
3162  *
3163  * @return
3164  *   0 on success, a negative errno value otherwise and rte_errno is set.
3165  */
3166 static int
3167 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
3168                                  const struct rte_flow_action *action,
3169                                  uint64_t action_flags __rte_unused,
3170                                  const struct rte_flow_attr *attr,
3171                                  struct rte_flow_error *error)
3172 {
3173         const struct rte_flow_action_set_meta *conf;
3174         uint32_t nic_mask = UINT32_MAX;
3175         int reg;
3176
3177         if (!mlx5_flow_ext_mreg_supported(dev))
3178                 return rte_flow_error_set(error, ENOTSUP,
3179                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3180                                           "extended metadata register"
3181                                           " isn't supported");
3182         reg = flow_dv_get_metadata_reg(dev, attr, error);
3183         if (reg < 0)
3184                 return reg;
3185         if (reg == REG_NON)
3186                 return rte_flow_error_set(error, ENOTSUP,
3187                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3188                                           "unavalable extended metadata register");
3189         if (reg != REG_A && reg != REG_B) {
3190                 struct mlx5_priv *priv = dev->data->dev_private;
3191
3192                 nic_mask = priv->sh->dv_meta_mask;
3193         }
3194         if (!(action->conf))
3195                 return rte_flow_error_set(error, EINVAL,
3196                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3197                                           "configuration cannot be null");
3198         conf = (const struct rte_flow_action_set_meta *)action->conf;
3199         if (!conf->mask)
3200                 return rte_flow_error_set(error, EINVAL,
3201                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3202                                           "zero mask doesn't have any effect");
3203         if (conf->mask & ~nic_mask)
3204                 return rte_flow_error_set(error, EINVAL,
3205                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3206                                           "meta data must be within reg C0");
3207         return 0;
3208 }
3209
3210 /**
3211  * Validate SET_TAG action.
3212  *
3213  * @param[in] dev
3214  *   Pointer to the rte_eth_dev structure.
3215  * @param[in] action
3216  *   Pointer to the action structure.
3217  * @param[in] action_flags
3218  *   Holds the actions detected until now.
3219  * @param[in] attr
3220  *   Pointer to flow attributes
3221  * @param[out] error
3222  *   Pointer to error structure.
3223  *
3224  * @return
3225  *   0 on success, a negative errno value otherwise and rte_errno is set.
3226  */
3227 static int
3228 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
3229                                 const struct rte_flow_action *action,
3230                                 uint64_t action_flags,
3231                                 const struct rte_flow_attr *attr,
3232                                 struct rte_flow_error *error)
3233 {
3234         const struct rte_flow_action_set_tag *conf;
3235         const uint64_t terminal_action_flags =
3236                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
3237                 MLX5_FLOW_ACTION_RSS;
3238         int ret;
3239
3240         if (!mlx5_flow_ext_mreg_supported(dev))
3241                 return rte_flow_error_set(error, ENOTSUP,
3242                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3243                                           "extensive metadata register"
3244                                           " isn't supported");
3245         if (!(action->conf))
3246                 return rte_flow_error_set(error, EINVAL,
3247                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3248                                           "configuration cannot be null");
3249         conf = (const struct rte_flow_action_set_tag *)action->conf;
3250         if (!conf->mask)
3251                 return rte_flow_error_set(error, EINVAL,
3252                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3253                                           "zero mask doesn't have any effect");
3254         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
3255         if (ret < 0)
3256                 return ret;
3257         if (!attr->transfer && attr->ingress &&
3258             (action_flags & terminal_action_flags))
3259                 return rte_flow_error_set(error, EINVAL,
3260                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3261                                           "set_tag has no effect"
3262                                           " with terminal actions");
3263         return 0;
3264 }
3265
3266 /**
3267  * Check if action counter is shared by either old or new mechanism.
3268  *
3269  * @param[in] action
3270  *   Pointer to the action structure.
3271  *
3272  * @return
3273  *   True when counter is shared, false otherwise.
3274  */
3275 static inline bool
3276 is_shared_action_count(const struct rte_flow_action *action)
3277 {
3278         const struct rte_flow_action_count *count =
3279                         (const struct rte_flow_action_count *)action->conf;
3280
3281         if ((int)action->type == MLX5_RTE_FLOW_ACTION_TYPE_COUNT)
3282                 return true;
3283         return !!(count && count->shared);
3284 }
3285
3286 /**
3287  * Validate count action.
3288  *
3289  * @param[in] dev
3290  *   Pointer to rte_eth_dev structure.
3291  * @param[in] shared
3292  *   Indicator if action is shared.
3293  * @param[in] action_flags
3294  *   Holds the actions detected until now.
3295  * @param[out] error
3296  *   Pointer to error structure.
3297  *
3298  * @return
3299  *   0 on success, a negative errno value otherwise and rte_errno is set.
3300  */
3301 static int
3302 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
3303                               uint64_t action_flags,
3304                               struct rte_flow_error *error)
3305 {
3306         struct mlx5_priv *priv = dev->data->dev_private;
3307
3308         if (!priv->config.devx)
3309                 goto notsup_err;
3310         if (action_flags & MLX5_FLOW_ACTION_COUNT)
3311                 return rte_flow_error_set(error, EINVAL,
3312                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3313                                           "duplicate count actions set");
3314         if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
3315             !priv->sh->flow_hit_aso_en)
3316                 return rte_flow_error_set(error, EINVAL,
3317                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3318                                           "old age and shared count combination is not supported");
3319 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
3320         return 0;
3321 #endif
3322 notsup_err:
3323         return rte_flow_error_set
3324                       (error, ENOTSUP,
3325                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3326                        NULL,
3327                        "count action not supported");
3328 }
3329
3330 /**
3331  * Validate the L2 encap action.
3332  *
3333  * @param[in] dev
3334  *   Pointer to the rte_eth_dev structure.
3335  * @param[in] action_flags
3336  *   Holds the actions detected until now.
3337  * @param[in] action
3338  *   Pointer to the action structure.
3339  * @param[in] attr
3340  *   Pointer to flow attributes.
3341  * @param[out] error
3342  *   Pointer to error structure.
3343  *
3344  * @return
3345  *   0 on success, a negative errno value otherwise and rte_errno is set.
3346  */
3347 static int
3348 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
3349                                  uint64_t action_flags,
3350                                  const struct rte_flow_action *action,
3351                                  const struct rte_flow_attr *attr,
3352                                  struct rte_flow_error *error)
3353 {
3354         const struct mlx5_priv *priv = dev->data->dev_private;
3355
3356         if (!(action->conf))
3357                 return rte_flow_error_set(error, EINVAL,
3358                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3359                                           "configuration cannot be null");
3360         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3361                 return rte_flow_error_set(error, EINVAL,
3362                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3363                                           "can only have a single encap action "
3364                                           "in a flow");
3365         if (!attr->transfer && priv->representor)
3366                 return rte_flow_error_set(error, ENOTSUP,
3367                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3368                                           "encap action for VF representor "
3369                                           "not supported on NIC table");
3370         return 0;
3371 }
3372
3373 /**
3374  * Validate a decap action.
3375  *
3376  * @param[in] dev
3377  *   Pointer to the rte_eth_dev structure.
3378  * @param[in] action_flags
3379  *   Holds the actions detected until now.
3380  * @param[in] action
3381  *   Pointer to the action structure.
3382  * @param[in] item_flags
3383  *   Holds the items detected.
3384  * @param[in] attr
3385  *   Pointer to flow attributes
3386  * @param[out] error
3387  *   Pointer to error structure.
3388  *
3389  * @return
3390  *   0 on success, a negative errno value otherwise and rte_errno is set.
3391  */
3392 static int
3393 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
3394                               uint64_t action_flags,
3395                               const struct rte_flow_action *action,
3396                               const uint64_t item_flags,
3397                               const struct rte_flow_attr *attr,
3398                               struct rte_flow_error *error)
3399 {
3400         const struct mlx5_priv *priv = dev->data->dev_private;
3401
3402         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
3403             !priv->config.decap_en)
3404                 return rte_flow_error_set(error, ENOTSUP,
3405                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3406                                           "decap is not enabled");
3407         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
3408                 return rte_flow_error_set(error, ENOTSUP,
3409                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3410                                           action_flags &
3411                                           MLX5_FLOW_ACTION_DECAP ? "can only "
3412                                           "have a single decap action" : "decap "
3413                                           "after encap is not supported");
3414         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
3415                 return rte_flow_error_set(error, EINVAL,
3416                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3417                                           "can't have decap action after"
3418                                           " modify action");
3419         if (attr->egress)
3420                 return rte_flow_error_set(error, ENOTSUP,
3421                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
3422                                           NULL,
3423                                           "decap action not supported for "
3424                                           "egress");
3425         if (!attr->transfer && priv->representor)
3426                 return rte_flow_error_set(error, ENOTSUP,
3427                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3428                                           "decap action for VF representor "
3429                                           "not supported on NIC table");
3430         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
3431             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
3432                 return rte_flow_error_set(error, ENOTSUP,
3433                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3434                                 "VXLAN item should be present for VXLAN decap");
3435         return 0;
3436 }
3437
3438 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
3439
3440 /**
3441  * Validate the raw encap and decap actions.
3442  *
3443  * @param[in] dev
3444  *   Pointer to the rte_eth_dev structure.
3445  * @param[in] decap
3446  *   Pointer to the decap action.
3447  * @param[in] encap
3448  *   Pointer to the encap action.
3449  * @param[in] attr
3450  *   Pointer to flow attributes
3451  * @param[in/out] action_flags
3452  *   Holds the actions detected until now.
3453  * @param[out] actions_n
3454  *   pointer to the number of actions counter.
3455  * @param[in] action
3456  *   Pointer to the action structure.
3457  * @param[in] item_flags
3458  *   Holds the items detected.
3459  * @param[out] error
3460  *   Pointer to error structure.
3461  *
3462  * @return
3463  *   0 on success, a negative errno value otherwise and rte_errno is set.
3464  */
3465 static int
3466 flow_dv_validate_action_raw_encap_decap
3467         (struct rte_eth_dev *dev,
3468          const struct rte_flow_action_raw_decap *decap,
3469          const struct rte_flow_action_raw_encap *encap,
3470          const struct rte_flow_attr *attr, uint64_t *action_flags,
3471          int *actions_n, const struct rte_flow_action *action,
3472          uint64_t item_flags, struct rte_flow_error *error)
3473 {
3474         const struct mlx5_priv *priv = dev->data->dev_private;
3475         int ret;
3476
3477         if (encap && (!encap->size || !encap->data))
3478                 return rte_flow_error_set(error, EINVAL,
3479                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3480                                           "raw encap data cannot be empty");
3481         if (decap && encap) {
3482                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
3483                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3484                         /* L3 encap. */
3485                         decap = NULL;
3486                 else if (encap->size <=
3487                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3488                            decap->size >
3489                            MLX5_ENCAPSULATION_DECISION_SIZE)
3490                         /* L3 decap. */
3491                         encap = NULL;
3492                 else if (encap->size >
3493                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3494                            decap->size >
3495                            MLX5_ENCAPSULATION_DECISION_SIZE)
3496                         /* 2 L2 actions: encap and decap. */
3497                         ;
3498                 else
3499                         return rte_flow_error_set(error,
3500                                 ENOTSUP,
3501                                 RTE_FLOW_ERROR_TYPE_ACTION,
3502                                 NULL, "unsupported too small "
3503                                 "raw decap and too small raw "
3504                                 "encap combination");
3505         }
3506         if (decap) {
3507                 ret = flow_dv_validate_action_decap(dev, *action_flags, action,
3508                                                     item_flags, attr, error);
3509                 if (ret < 0)
3510                         return ret;
3511                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
3512                 ++(*actions_n);
3513         }
3514         if (encap) {
3515                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
3516                         return rte_flow_error_set(error, ENOTSUP,
3517                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3518                                                   NULL,
3519                                                   "small raw encap size");
3520                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
3521                         return rte_flow_error_set(error, EINVAL,
3522                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3523                                                   NULL,
3524                                                   "more than one encap action");
3525                 if (!attr->transfer && priv->representor)
3526                         return rte_flow_error_set
3527                                         (error, ENOTSUP,
3528                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3529                                          "encap action for VF representor "
3530                                          "not supported on NIC table");
3531                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
3532                 ++(*actions_n);
3533         }
3534         return 0;
3535 }
3536
3537 /*
3538  * Validate the ASO CT action.
3539  *
3540  * @param[in] dev
3541  *   Pointer to the rte_eth_dev structure.
3542  * @param[in] action_flags
3543  *   Holds the actions detected until now.
3544  * @param[in] item_flags
3545  *   The items found in this flow rule.
3546  * @param[in] attr
3547  *   Pointer to flow attributes.
3548  * @param[out] error
3549  *   Pointer to error structure.
3550  *
3551  * @return
3552  *   0 on success, a negative errno value otherwise and rte_errno is set.
3553  */
3554 static int
3555 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
3556                                uint64_t action_flags,
3557                                uint64_t item_flags,
3558                                const struct rte_flow_attr *attr,
3559                                struct rte_flow_error *error)
3560 {
3561         RTE_SET_USED(dev);
3562
3563         if (attr->group == 0 && !attr->transfer)
3564                 return rte_flow_error_set(error, ENOTSUP,
3565                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3566                                           NULL,
3567                                           "Only support non-root table");
3568         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
3569                 return rte_flow_error_set(error, ENOTSUP,
3570                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3571                                           "CT cannot follow a fate action");
3572         if ((action_flags & MLX5_FLOW_ACTION_METER) ||
3573             (action_flags & MLX5_FLOW_ACTION_AGE))
3574                 return rte_flow_error_set(error, EINVAL,
3575                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3576                                           "Only one ASO action is supported");
3577         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3578                 return rte_flow_error_set(error, EINVAL,
3579                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3580                                           "Encap cannot exist before CT");
3581         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3582                 return rte_flow_error_set(error, EINVAL,
3583                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3584                                           "Not a outer TCP packet");
3585         return 0;
3586 }
3587
3588 /**
3589  * Match encap_decap resource.
3590  *
3591  * @param list
3592  *   Pointer to the hash list.
3593  * @param entry
3594  *   Pointer to exist resource entry object.
3595  * @param key
3596  *   Key of the new entry.
3597  * @param ctx_cb
3598  *   Pointer to new encap_decap resource.
3599  *
3600  * @return
3601  *   0 on matching, none-zero otherwise.
3602  */
3603 int
3604 flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
3605                              struct mlx5_hlist_entry *entry,
3606                              uint64_t key __rte_unused, void *cb_ctx)
3607 {
3608         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3609         struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
3610         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3611
3612         cache_resource = container_of(entry,
3613                                       struct mlx5_flow_dv_encap_decap_resource,
3614                                       entry);
3615         if (resource->reformat_type == cache_resource->reformat_type &&
3616             resource->ft_type == cache_resource->ft_type &&
3617             resource->flags == cache_resource->flags &&
3618             resource->size == cache_resource->size &&
3619             !memcmp((const void *)resource->buf,
3620                     (const void *)cache_resource->buf,
3621                     resource->size))
3622                 return 0;
3623         return -1;
3624 }
3625
3626 /**
3627  * Allocate encap_decap resource.
3628  *
3629  * @param list
3630  *   Pointer to the hash list.
3631  * @param entry
3632  *   Pointer to exist resource entry object.
3633  * @param ctx_cb
3634  *   Pointer to new encap_decap resource.
3635  *
3636  * @return
3637  *   0 on matching, none-zero otherwise.
3638  */
3639 struct mlx5_hlist_entry *
3640 flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
3641                               uint64_t key __rte_unused,
3642                               void *cb_ctx)
3643 {
3644         struct mlx5_dev_ctx_shared *sh = list->ctx;
3645         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3646         struct mlx5dv_dr_domain *domain;
3647         struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
3648         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3649         uint32_t idx;
3650         int ret;
3651
3652         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3653                 domain = sh->fdb_domain;
3654         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3655                 domain = sh->rx_domain;
3656         else
3657                 domain = sh->tx_domain;
3658         /* Register new encap/decap resource. */
3659         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
3660                                        &idx);
3661         if (!cache_resource) {
3662                 rte_flow_error_set(ctx->error, ENOMEM,
3663                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3664                                    "cannot allocate resource memory");
3665                 return NULL;
3666         }
3667         *cache_resource = *resource;
3668         cache_resource->idx = idx;
3669         ret = mlx5_flow_os_create_flow_action_packet_reformat
3670                                         (sh->ctx, domain, cache_resource,
3671                                          &cache_resource->action);
3672         if (ret) {
3673                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
3674                 rte_flow_error_set(ctx->error, ENOMEM,
3675                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3676                                    NULL, "cannot create action");
3677                 return NULL;
3678         }
3679
3680         return &cache_resource->entry;
3681 }
3682
3683 /**
3684  * Find existing encap/decap resource or create and register a new one.
3685  *
3686  * @param[in, out] dev
3687  *   Pointer to rte_eth_dev structure.
3688  * @param[in, out] resource
3689  *   Pointer to encap/decap resource.
3690  * @parm[in, out] dev_flow
3691  *   Pointer to the dev_flow.
3692  * @param[out] error
3693  *   pointer to error structure.
3694  *
3695  * @return
3696  *   0 on success otherwise -errno and errno is set.
3697  */
3698 static int
3699 flow_dv_encap_decap_resource_register
3700                         (struct rte_eth_dev *dev,
3701                          struct mlx5_flow_dv_encap_decap_resource *resource,
3702                          struct mlx5_flow *dev_flow,
3703                          struct rte_flow_error *error)
3704 {
3705         struct mlx5_priv *priv = dev->data->dev_private;
3706         struct mlx5_dev_ctx_shared *sh = priv->sh;
3707         struct mlx5_hlist_entry *entry;
3708         union {
3709                 struct {
3710                         uint32_t ft_type:8;
3711                         uint32_t refmt_type:8;
3712                         /*
3713                          * Header reformat actions can be shared between
3714                          * non-root tables. One bit to indicate non-root
3715                          * table or not.
3716                          */
3717                         uint32_t is_root:1;
3718                         uint32_t reserve:15;
3719                 };
3720                 uint32_t v32;
3721         } encap_decap_key = {
3722                 {
3723                         .ft_type = resource->ft_type,
3724                         .refmt_type = resource->reformat_type,
3725                         .is_root = !!dev_flow->dv.group,
3726                         .reserve = 0,
3727                 }
3728         };
3729         struct mlx5_flow_cb_ctx ctx = {
3730                 .error = error,
3731                 .data = resource,
3732         };
3733         uint64_t key64;
3734
3735         resource->flags = dev_flow->dv.group ? 0 : 1;
3736         key64 =  __rte_raw_cksum(&encap_decap_key.v32,
3737                                  sizeof(encap_decap_key.v32), 0);
3738         if (resource->reformat_type !=
3739             MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
3740             resource->size)
3741                 key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
3742         entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
3743         if (!entry)
3744                 return -rte_errno;
3745         resource = container_of(entry, typeof(*resource), entry);
3746         dev_flow->dv.encap_decap = resource;
3747         dev_flow->handle->dvh.rix_encap_decap = resource->idx;
3748         return 0;
3749 }
3750
3751 /**
3752  * Find existing table jump resource or create and register a new one.
3753  *
3754  * @param[in, out] dev
3755  *   Pointer to rte_eth_dev structure.
3756  * @param[in, out] tbl
3757  *   Pointer to flow table resource.
3758  * @parm[in, out] dev_flow
3759  *   Pointer to the dev_flow.
3760  * @param[out] error
3761  *   pointer to error structure.
3762  *
3763  * @return
3764  *   0 on success otherwise -errno and errno is set.
3765  */
3766 static int
3767 flow_dv_jump_tbl_resource_register
3768                         (struct rte_eth_dev *dev __rte_unused,
3769                          struct mlx5_flow_tbl_resource *tbl,
3770                          struct mlx5_flow *dev_flow,
3771                          struct rte_flow_error *error __rte_unused)
3772 {
3773         struct mlx5_flow_tbl_data_entry *tbl_data =
3774                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
3775
3776         MLX5_ASSERT(tbl);
3777         MLX5_ASSERT(tbl_data->jump.action);
3778         dev_flow->handle->rix_jump = tbl_data->idx;
3779         dev_flow->dv.jump = &tbl_data->jump;
3780         return 0;
3781 }
3782
3783 int
3784 flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
3785                          struct mlx5_cache_entry *entry, void *cb_ctx)
3786 {
3787         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3788         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3789         struct mlx5_flow_dv_port_id_action_resource *res =
3790                         container_of(entry, typeof(*res), entry);
3791
3792         return ref->port_id != res->port_id;
3793 }
3794
3795 struct mlx5_cache_entry *
3796 flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
3797                           struct mlx5_cache_entry *entry __rte_unused,
3798                           void *cb_ctx)
3799 {
3800         struct mlx5_dev_ctx_shared *sh = list->ctx;
3801         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3802         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3803         struct mlx5_flow_dv_port_id_action_resource *cache;
3804         uint32_t idx;
3805         int ret;
3806
3807         /* Register new port id action resource. */
3808         cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3809         if (!cache) {
3810                 rte_flow_error_set(ctx->error, ENOMEM,
3811                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3812                                    "cannot allocate port_id action cache memory");
3813                 return NULL;
3814         }
3815         *cache = *ref;
3816         ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
3817                                                         ref->port_id,
3818                                                         &cache->action);
3819         if (ret) {
3820                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
3821                 rte_flow_error_set(ctx->error, ENOMEM,
3822                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3823                                    "cannot create action");
3824                 return NULL;
3825         }
3826         cache->idx = idx;
3827         return &cache->entry;
3828 }
3829
3830 /**
3831  * Find existing table port ID resource or create and register a new one.
3832  *
3833  * @param[in, out] dev
3834  *   Pointer to rte_eth_dev structure.
3835  * @param[in, out] resource
3836  *   Pointer to port ID action resource.
3837  * @parm[in, out] dev_flow
3838  *   Pointer to the dev_flow.
3839  * @param[out] error
3840  *   pointer to error structure.
3841  *
3842  * @return
3843  *   0 on success otherwise -errno and errno is set.
3844  */
3845 static int
3846 flow_dv_port_id_action_resource_register
3847                         (struct rte_eth_dev *dev,
3848                          struct mlx5_flow_dv_port_id_action_resource *resource,
3849                          struct mlx5_flow *dev_flow,
3850                          struct rte_flow_error *error)
3851 {
3852         struct mlx5_priv *priv = dev->data->dev_private;
3853         struct mlx5_cache_entry *entry;
3854         struct mlx5_flow_dv_port_id_action_resource *cache;
3855         struct mlx5_flow_cb_ctx ctx = {
3856                 .error = error,
3857                 .data = resource,
3858         };
3859
3860         entry = mlx5_cache_register(&priv->sh->port_id_action_list, &ctx);
3861         if (!entry)
3862                 return -rte_errno;
3863         cache = container_of(entry, typeof(*cache), entry);
3864         dev_flow->dv.port_id_action = cache;
3865         dev_flow->handle->rix_port_id_action = cache->idx;
3866         return 0;
3867 }
3868
3869 int
3870 flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
3871                          struct mlx5_cache_entry *entry, void *cb_ctx)
3872 {
3873         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3874         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3875         struct mlx5_flow_dv_push_vlan_action_resource *res =
3876                         container_of(entry, typeof(*res), entry);
3877
3878         return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
3879 }
3880
3881 struct mlx5_cache_entry *
3882 flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
3883                           struct mlx5_cache_entry *entry __rte_unused,
3884                           void *cb_ctx)
3885 {
3886         struct mlx5_dev_ctx_shared *sh = list->ctx;
3887         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3888         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3889         struct mlx5_flow_dv_push_vlan_action_resource *cache;
3890         struct mlx5dv_dr_domain *domain;
3891         uint32_t idx;
3892         int ret;
3893
3894         /* Register new port id action resource. */
3895         cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3896         if (!cache) {
3897                 rte_flow_error_set(ctx->error, ENOMEM,
3898                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3899                                    "cannot allocate push_vlan action cache memory");
3900                 return NULL;
3901         }
3902         *cache = *ref;
3903         if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3904                 domain = sh->fdb_domain;
3905         else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3906                 domain = sh->rx_domain;
3907         else
3908                 domain = sh->tx_domain;
3909         ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
3910                                                         &cache->action);
3911         if (ret) {
3912                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
3913                 rte_flow_error_set(ctx->error, ENOMEM,
3914                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3915                                    "cannot create push vlan action");
3916                 return NULL;
3917         }
3918         cache->idx = idx;
3919         return &cache->entry;
3920 }
3921
3922 /**
3923  * Find existing push vlan resource or create and register a new one.
3924  *
3925  * @param [in, out] dev
3926  *   Pointer to rte_eth_dev structure.
3927  * @param[in, out] resource
3928  *   Pointer to port ID action resource.
3929  * @parm[in, out] dev_flow
3930  *   Pointer to the dev_flow.
3931  * @param[out] error
3932  *   pointer to error structure.
3933  *
3934  * @return
3935  *   0 on success otherwise -errno and errno is set.
3936  */
3937 static int
3938 flow_dv_push_vlan_action_resource_register
3939                        (struct rte_eth_dev *dev,
3940                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
3941                         struct mlx5_flow *dev_flow,
3942                         struct rte_flow_error *error)
3943 {
3944         struct mlx5_priv *priv = dev->data->dev_private;
3945         struct mlx5_flow_dv_push_vlan_action_resource *cache;
3946         struct mlx5_cache_entry *entry;
3947         struct mlx5_flow_cb_ctx ctx = {
3948                 .error = error,
3949                 .data = resource,
3950         };
3951
3952         entry = mlx5_cache_register(&priv->sh->push_vlan_action_list, &ctx);
3953         if (!entry)
3954                 return -rte_errno;
3955         cache = container_of(entry, typeof(*cache), entry);
3956
3957         dev_flow->handle->dvh.rix_push_vlan = cache->idx;
3958         dev_flow->dv.push_vlan_res = cache;
3959         return 0;
3960 }
3961
3962 /**
3963  * Get the size of specific rte_flow_item_type hdr size
3964  *
3965  * @param[in] item_type
3966  *   Tested rte_flow_item_type.
3967  *
3968  * @return
3969  *   sizeof struct item_type, 0 if void or irrelevant.
3970  */
3971 static size_t
3972 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3973 {
3974         size_t retval;
3975
3976         switch (item_type) {
3977         case RTE_FLOW_ITEM_TYPE_ETH:
3978                 retval = sizeof(struct rte_ether_hdr);
3979                 break;
3980         case RTE_FLOW_ITEM_TYPE_VLAN:
3981                 retval = sizeof(struct rte_vlan_hdr);
3982                 break;
3983         case RTE_FLOW_ITEM_TYPE_IPV4:
3984                 retval = sizeof(struct rte_ipv4_hdr);
3985                 break;
3986         case RTE_FLOW_ITEM_TYPE_IPV6:
3987                 retval = sizeof(struct rte_ipv6_hdr);
3988                 break;
3989         case RTE_FLOW_ITEM_TYPE_UDP:
3990                 retval = sizeof(struct rte_udp_hdr);
3991                 break;
3992         case RTE_FLOW_ITEM_TYPE_TCP:
3993                 retval = sizeof(struct rte_tcp_hdr);
3994                 break;
3995         case RTE_FLOW_ITEM_TYPE_VXLAN:
3996         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3997                 retval = sizeof(struct rte_vxlan_hdr);
3998                 break;
3999         case RTE_FLOW_ITEM_TYPE_GRE:
4000         case RTE_FLOW_ITEM_TYPE_NVGRE:
4001                 retval = sizeof(struct rte_gre_hdr);
4002                 break;
4003         case RTE_FLOW_ITEM_TYPE_MPLS:
4004                 retval = sizeof(struct rte_mpls_hdr);
4005                 break;
4006         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
4007         default:
4008                 retval = 0;
4009                 break;
4010         }
4011         return retval;
4012 }
4013
4014 #define MLX5_ENCAP_IPV4_VERSION         0x40
4015 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
4016 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
4017 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
4018 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
4019 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
4020 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
4021
4022 /**
4023  * Convert the encap action data from list of rte_flow_item to raw buffer
4024  *
4025  * @param[in] items
4026  *   Pointer to rte_flow_item objects list.
4027  * @param[out] buf
4028  *   Pointer to the output buffer.
4029  * @param[out] size
4030  *   Pointer to the output buffer size.
4031  * @param[out] error
4032  *   Pointer to the error structure.
4033  *
4034  * @return
4035  *   0 on success, a negative errno value otherwise and rte_errno is set.
4036  */
4037 static int
4038 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
4039                            size_t *size, struct rte_flow_error *error)
4040 {
4041         struct rte_ether_hdr *eth = NULL;
4042         struct rte_vlan_hdr *vlan = NULL;
4043         struct rte_ipv4_hdr *ipv4 = NULL;
4044         struct rte_ipv6_hdr *ipv6 = NULL;
4045         struct rte_udp_hdr *udp = NULL;
4046         struct rte_vxlan_hdr *vxlan = NULL;
4047         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
4048         struct rte_gre_hdr *gre = NULL;
4049         size_t len;
4050         size_t temp_size = 0;
4051
4052         if (!items)
4053                 return rte_flow_error_set(error, EINVAL,
4054                                           RTE_FLOW_ERROR_TYPE_ACTION,
4055                                           NULL, "invalid empty data");
4056         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4057                 len = flow_dv_get_item_hdr_len(items->type);
4058                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
4059                         return rte_flow_error_set(error, EINVAL,
4060                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4061                                                   (void *)items->type,
4062                                                   "items total size is too big"
4063                                                   " for encap action");
4064                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
4065                 switch (items->type) {
4066                 case RTE_FLOW_ITEM_TYPE_ETH:
4067                         eth = (struct rte_ether_hdr *)&buf[temp_size];
4068                         break;
4069                 case RTE_FLOW_ITEM_TYPE_VLAN:
4070                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
4071                         if (!eth)
4072                                 return rte_flow_error_set(error, EINVAL,
4073                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4074                                                 (void *)items->type,
4075                                                 "eth header not found");
4076                         if (!eth->ether_type)
4077                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
4078                         break;
4079                 case RTE_FLOW_ITEM_TYPE_IPV4:
4080                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
4081                         if (!vlan && !eth)
4082                                 return rte_flow_error_set(error, EINVAL,
4083                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4084                                                 (void *)items->type,
4085                                                 "neither eth nor vlan"
4086                                                 " header found");
4087                         if (vlan && !vlan->eth_proto)
4088                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4089                         else if (eth && !eth->ether_type)
4090                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4091                         if (!ipv4->version_ihl)
4092                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
4093                                                     MLX5_ENCAP_IPV4_IHL_MIN;
4094                         if (!ipv4->time_to_live)
4095                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
4096                         break;
4097                 case RTE_FLOW_ITEM_TYPE_IPV6:
4098                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
4099                         if (!vlan && !eth)
4100                                 return rte_flow_error_set(error, EINVAL,
4101                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4102                                                 (void *)items->type,
4103                                                 "neither eth nor vlan"
4104                                                 " header found");
4105                         if (vlan && !vlan->eth_proto)
4106                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4107                         else if (eth && !eth->ether_type)
4108                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4109                         if (!ipv6->vtc_flow)
4110                                 ipv6->vtc_flow =
4111                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
4112                         if (!ipv6->hop_limits)
4113                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
4114                         break;
4115                 case RTE_FLOW_ITEM_TYPE_UDP:
4116                         udp = (struct rte_udp_hdr *)&buf[temp_size];
4117                         if (!ipv4 && !ipv6)
4118                                 return rte_flow_error_set(error, EINVAL,
4119                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4120                                                 (void *)items->type,
4121                                                 "ip header not found");
4122                         if (ipv4 && !ipv4->next_proto_id)
4123                                 ipv4->next_proto_id = IPPROTO_UDP;
4124                         else if (ipv6 && !ipv6->proto)
4125                                 ipv6->proto = IPPROTO_UDP;
4126                         break;
4127                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4128                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
4129                         if (!udp)
4130                                 return rte_flow_error_set(error, EINVAL,
4131                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4132                                                 (void *)items->type,
4133                                                 "udp header not found");
4134                         if (!udp->dst_port)
4135                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
4136                         if (!vxlan->vx_flags)
4137                                 vxlan->vx_flags =
4138                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
4139                         break;
4140                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4141                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
4142                         if (!udp)
4143                                 return rte_flow_error_set(error, EINVAL,
4144                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4145                                                 (void *)items->type,
4146                                                 "udp header not found");
4147                         if (!vxlan_gpe->proto)
4148                                 return rte_flow_error_set(error, EINVAL,
4149                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4150                                                 (void *)items->type,
4151                                                 "next protocol not found");
4152                         if (!udp->dst_port)
4153                                 udp->dst_port =
4154                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
4155                         if (!vxlan_gpe->vx_flags)
4156                                 vxlan_gpe->vx_flags =
4157                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
4158                         break;
4159                 case RTE_FLOW_ITEM_TYPE_GRE:
4160                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4161                         gre = (struct rte_gre_hdr *)&buf[temp_size];
4162                         if (!gre->proto)
4163                                 return rte_flow_error_set(error, EINVAL,
4164                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4165                                                 (void *)items->type,
4166                                                 "next protocol not found");
4167                         if (!ipv4 && !ipv6)
4168                                 return rte_flow_error_set(error, EINVAL,
4169                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4170                                                 (void *)items->type,
4171                                                 "ip header not found");
4172                         if (ipv4 && !ipv4->next_proto_id)
4173                                 ipv4->next_proto_id = IPPROTO_GRE;
4174                         else if (ipv6 && !ipv6->proto)
4175                                 ipv6->proto = IPPROTO_GRE;
4176                         break;
4177                 case RTE_FLOW_ITEM_TYPE_VOID:
4178                         break;
4179                 default:
4180                         return rte_flow_error_set(error, EINVAL,
4181                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4182                                                   (void *)items->type,
4183                                                   "unsupported item type");
4184                         break;
4185                 }
4186                 temp_size += len;
4187         }
4188         *size = temp_size;
4189         return 0;
4190 }
4191
4192 static int
4193 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
4194 {
4195         struct rte_ether_hdr *eth = NULL;
4196         struct rte_vlan_hdr *vlan = NULL;
4197         struct rte_ipv6_hdr *ipv6 = NULL;
4198         struct rte_udp_hdr *udp = NULL;
4199         char *next_hdr;
4200         uint16_t proto;
4201
4202         eth = (struct rte_ether_hdr *)data;
4203         next_hdr = (char *)(eth + 1);
4204         proto = RTE_BE16(eth->ether_type);
4205
4206         /* VLAN skipping */
4207         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
4208                 vlan = (struct rte_vlan_hdr *)next_hdr;
4209                 proto = RTE_BE16(vlan->eth_proto);
4210                 next_hdr += sizeof(struct rte_vlan_hdr);
4211         }
4212
4213         /* HW calculates IPv4 csum. no need to proceed */
4214         if (proto == RTE_ETHER_TYPE_IPV4)
4215                 return 0;
4216
4217         /* non IPv4/IPv6 header. not supported */
4218         if (proto != RTE_ETHER_TYPE_IPV6) {
4219                 return rte_flow_error_set(error, ENOTSUP,
4220                                           RTE_FLOW_ERROR_TYPE_ACTION,
4221                                           NULL, "Cannot offload non IPv4/IPv6");
4222         }
4223
4224         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
4225
4226         /* ignore non UDP */
4227         if (ipv6->proto != IPPROTO_UDP)
4228                 return 0;
4229
4230         udp = (struct rte_udp_hdr *)(ipv6 + 1);
4231         udp->dgram_cksum = 0;
4232
4233         return 0;
4234 }
4235
4236 /**
4237  * Convert L2 encap action to DV specification.
4238  *
4239  * @param[in] dev
4240  *   Pointer to rte_eth_dev structure.
4241  * @param[in] action
4242  *   Pointer to action structure.
4243  * @param[in, out] dev_flow
4244  *   Pointer to the mlx5_flow.
4245  * @param[in] transfer
4246  *   Mark if the flow is E-Switch flow.
4247  * @param[out] error
4248  *   Pointer to the error structure.
4249  *
4250  * @return
4251  *   0 on success, a negative errno value otherwise and rte_errno is set.
4252  */
4253 static int
4254 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
4255                                const struct rte_flow_action *action,
4256                                struct mlx5_flow *dev_flow,
4257                                uint8_t transfer,
4258                                struct rte_flow_error *error)
4259 {
4260         const struct rte_flow_item *encap_data;
4261         const struct rte_flow_action_raw_encap *raw_encap_data;
4262         struct mlx5_flow_dv_encap_decap_resource res = {
4263                 .reformat_type =
4264                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
4265                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4266                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
4267         };
4268
4269         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4270                 raw_encap_data =
4271                         (const struct rte_flow_action_raw_encap *)action->conf;
4272                 res.size = raw_encap_data->size;
4273                 memcpy(res.buf, raw_encap_data->data, res.size);
4274         } else {
4275                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
4276                         encap_data =
4277                                 ((const struct rte_flow_action_vxlan_encap *)
4278                                                 action->conf)->definition;
4279                 else
4280                         encap_data =
4281                                 ((const struct rte_flow_action_nvgre_encap *)
4282                                                 action->conf)->definition;
4283                 if (flow_dv_convert_encap_data(encap_data, res.buf,
4284                                                &res.size, error))
4285                         return -rte_errno;
4286         }
4287         if (flow_dv_zero_encap_udp_csum(res.buf, error))
4288                 return -rte_errno;
4289         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4290                 return rte_flow_error_set(error, EINVAL,
4291                                           RTE_FLOW_ERROR_TYPE_ACTION,
4292                                           NULL, "can't create L2 encap action");
4293         return 0;
4294 }
4295
4296 /**
4297  * Convert L2 decap action to DV specification.
4298  *
4299  * @param[in] dev
4300  *   Pointer to rte_eth_dev structure.
4301  * @param[in, out] dev_flow
4302  *   Pointer to the mlx5_flow.
4303  * @param[in] transfer
4304  *   Mark if the flow is E-Switch flow.
4305  * @param[out] error
4306  *   Pointer to the error structure.
4307  *
4308  * @return
4309  *   0 on success, a negative errno value otherwise and rte_errno is set.
4310  */
4311 static int
4312 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
4313                                struct mlx5_flow *dev_flow,
4314                                uint8_t transfer,
4315                                struct rte_flow_error *error)
4316 {
4317         struct mlx5_flow_dv_encap_decap_resource res = {
4318                 .size = 0,
4319                 .reformat_type =
4320                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
4321                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4322                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
4323         };
4324
4325         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4326                 return rte_flow_error_set(error, EINVAL,
4327                                           RTE_FLOW_ERROR_TYPE_ACTION,
4328                                           NULL, "can't create L2 decap action");
4329         return 0;
4330 }
4331
4332 /**
4333  * Convert raw decap/encap (L3 tunnel) action to DV specification.
4334  *
4335  * @param[in] dev
4336  *   Pointer to rte_eth_dev structure.
4337  * @param[in] action
4338  *   Pointer to action structure.
4339  * @param[in, out] dev_flow
4340  *   Pointer to the mlx5_flow.
4341  * @param[in] attr
4342  *   Pointer to the flow attributes.
4343  * @param[out] error
4344  *   Pointer to the error structure.
4345  *
4346  * @return
4347  *   0 on success, a negative errno value otherwise and rte_errno is set.
4348  */
4349 static int
4350 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
4351                                 const struct rte_flow_action *action,
4352                                 struct mlx5_flow *dev_flow,
4353                                 const struct rte_flow_attr *attr,
4354                                 struct rte_flow_error *error)
4355 {
4356         const struct rte_flow_action_raw_encap *encap_data;
4357         struct mlx5_flow_dv_encap_decap_resource res;
4358
4359         memset(&res, 0, sizeof(res));
4360         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
4361         res.size = encap_data->size;
4362         memcpy(res.buf, encap_data->data, res.size);
4363         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
4364                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
4365                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
4366         if (attr->transfer)
4367                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4368         else
4369                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4370                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4371         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4372                 return rte_flow_error_set(error, EINVAL,
4373                                           RTE_FLOW_ERROR_TYPE_ACTION,
4374                                           NULL, "can't create encap action");
4375         return 0;
4376 }
4377
4378 /**
4379  * Create action push VLAN.
4380  *
4381  * @param[in] dev
4382  *   Pointer to rte_eth_dev structure.
4383  * @param[in] attr
4384  *   Pointer to the flow attributes.
4385  * @param[in] vlan
4386  *   Pointer to the vlan to push to the Ethernet header.
4387  * @param[in, out] dev_flow
4388  *   Pointer to the mlx5_flow.
4389  * @param[out] error
4390  *   Pointer to the error structure.
4391  *
4392  * @return
4393  *   0 on success, a negative errno value otherwise and rte_errno is set.
4394  */
4395 static int
4396 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
4397                                 const struct rte_flow_attr *attr,
4398                                 const struct rte_vlan_hdr *vlan,
4399                                 struct mlx5_flow *dev_flow,
4400                                 struct rte_flow_error *error)
4401 {
4402         struct mlx5_flow_dv_push_vlan_action_resource res;
4403
4404         memset(&res, 0, sizeof(res));
4405         res.vlan_tag =
4406                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
4407                                  vlan->vlan_tci);
4408         if (attr->transfer)
4409                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4410         else
4411                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4412                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4413         return flow_dv_push_vlan_action_resource_register
4414                                             (dev, &res, dev_flow, error);
4415 }
4416
4417 /**
4418  * Validate the modify-header actions.
4419  *
4420  * @param[in] action_flags
4421  *   Holds the actions detected until now.
4422  * @param[in] action
4423  *   Pointer to the modify action.
4424  * @param[out] error
4425  *   Pointer to error structure.
4426  *
4427  * @return
4428  *   0 on success, a negative errno value otherwise and rte_errno is set.
4429  */
4430 static int
4431 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
4432                                    const struct rte_flow_action *action,
4433                                    struct rte_flow_error *error)
4434 {
4435         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
4436                 return rte_flow_error_set(error, EINVAL,
4437                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4438                                           NULL, "action configuration not set");
4439         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
4440                 return rte_flow_error_set(error, EINVAL,
4441                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4442                                           "can't have encap action before"
4443                                           " modify action");
4444         return 0;
4445 }
4446
4447 /**
4448  * Validate the modify-header MAC address actions.
4449  *
4450  * @param[in] action_flags
4451  *   Holds the actions detected until now.
4452  * @param[in] action
4453  *   Pointer to the modify action.
4454  * @param[in] item_flags
4455  *   Holds the items detected.
4456  * @param[out] error
4457  *   Pointer to error structure.
4458  *
4459  * @return
4460  *   0 on success, a negative errno value otherwise and rte_errno is set.
4461  */
4462 static int
4463 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
4464                                    const struct rte_flow_action *action,
4465                                    const uint64_t item_flags,
4466                                    struct rte_flow_error *error)
4467 {
4468         int ret = 0;
4469
4470         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4471         if (!ret) {
4472                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
4473                         return rte_flow_error_set(error, EINVAL,
4474                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4475                                                   NULL,
4476                                                   "no L2 item in pattern");
4477         }
4478         return ret;
4479 }
4480
4481 /**
4482  * Validate the modify-header IPv4 address actions.
4483  *
4484  * @param[in] action_flags
4485  *   Holds the actions detected until now.
4486  * @param[in] action
4487  *   Pointer to the modify action.
4488  * @param[in] item_flags
4489  *   Holds the items detected.
4490  * @param[out] error
4491  *   Pointer to error structure.
4492  *
4493  * @return
4494  *   0 on success, a negative errno value otherwise and rte_errno is set.
4495  */
4496 static int
4497 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
4498                                     const struct rte_flow_action *action,
4499                                     const uint64_t item_flags,
4500                                     struct rte_flow_error *error)
4501 {
4502         int ret = 0;
4503         uint64_t layer;
4504
4505         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4506         if (!ret) {
4507                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4508                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4509                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4510                 if (!(item_flags & layer))
4511                         return rte_flow_error_set(error, EINVAL,
4512                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4513                                                   NULL,
4514                                                   "no ipv4 item in pattern");
4515         }
4516         return ret;
4517 }
4518
4519 /**
4520  * Validate the modify-header IPv6 address actions.
4521  *
4522  * @param[in] action_flags
4523  *   Holds the actions detected until now.
4524  * @param[in] action
4525  *   Pointer to the modify action.
4526  * @param[in] item_flags
4527  *   Holds the items detected.
4528  * @param[out] error
4529  *   Pointer to error structure.
4530  *
4531  * @return
4532  *   0 on success, a negative errno value otherwise and rte_errno is set.
4533  */
4534 static int
4535 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
4536                                     const struct rte_flow_action *action,
4537                                     const uint64_t item_flags,
4538                                     struct rte_flow_error *error)
4539 {
4540         int ret = 0;
4541         uint64_t layer;
4542
4543         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4544         if (!ret) {
4545                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4546                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4547                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4548                 if (!(item_flags & layer))
4549                         return rte_flow_error_set(error, EINVAL,
4550                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4551                                                   NULL,
4552                                                   "no ipv6 item in pattern");
4553         }
4554         return ret;
4555 }
4556
4557 /**
4558  * Validate the modify-header TP actions.
4559  *
4560  * @param[in] action_flags
4561  *   Holds the actions detected until now.
4562  * @param[in] action
4563  *   Pointer to the modify action.
4564  * @param[in] item_flags
4565  *   Holds the items detected.
4566  * @param[out] error
4567  *   Pointer to error structure.
4568  *
4569  * @return
4570  *   0 on success, a negative errno value otherwise and rte_errno is set.
4571  */
4572 static int
4573 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
4574                                   const struct rte_flow_action *action,
4575                                   const uint64_t item_flags,
4576                                   struct rte_flow_error *error)
4577 {
4578         int ret = 0;
4579         uint64_t layer;
4580
4581         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4582         if (!ret) {
4583                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4584                                  MLX5_FLOW_LAYER_INNER_L4 :
4585                                  MLX5_FLOW_LAYER_OUTER_L4;
4586                 if (!(item_flags & layer))
4587                         return rte_flow_error_set(error, EINVAL,
4588                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4589                                                   NULL, "no transport layer "
4590                                                   "in pattern");
4591         }
4592         return ret;
4593 }
4594
4595 /**
4596  * Validate the modify-header actions of increment/decrement
4597  * TCP Sequence-number.
4598  *
4599  * @param[in] action_flags
4600  *   Holds the actions detected until now.
4601  * @param[in] action
4602  *   Pointer to the modify action.
4603  * @param[in] item_flags
4604  *   Holds the items detected.
4605  * @param[out] error
4606  *   Pointer to error structure.
4607  *
4608  * @return
4609  *   0 on success, a negative errno value otherwise and rte_errno is set.
4610  */
4611 static int
4612 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
4613                                        const struct rte_flow_action *action,
4614                                        const uint64_t item_flags,
4615                                        struct rte_flow_error *error)
4616 {
4617         int ret = 0;
4618         uint64_t layer;
4619
4620         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4621         if (!ret) {
4622                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4623                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4624                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4625                 if (!(item_flags & layer))
4626                         return rte_flow_error_set(error, EINVAL,
4627                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4628                                                   NULL, "no TCP item in"
4629                                                   " pattern");
4630                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
4631                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
4632                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
4633                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
4634                         return rte_flow_error_set(error, EINVAL,
4635                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4636                                                   NULL,
4637                                                   "cannot decrease and increase"
4638                                                   " TCP sequence number"
4639                                                   " at the same time");
4640         }
4641         return ret;
4642 }
4643
4644 /**
4645  * Validate the modify-header actions of increment/decrement
4646  * TCP Acknowledgment number.
4647  *
4648  * @param[in] action_flags
4649  *   Holds the actions detected until now.
4650  * @param[in] action
4651  *   Pointer to the modify action.
4652  * @param[in] item_flags
4653  *   Holds the items detected.
4654  * @param[out] error
4655  *   Pointer to error structure.
4656  *
4657  * @return
4658  *   0 on success, a negative errno value otherwise and rte_errno is set.
4659  */
4660 static int
4661 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
4662                                        const struct rte_flow_action *action,
4663                                        const uint64_t item_flags,
4664                                        struct rte_flow_error *error)
4665 {
4666         int ret = 0;
4667         uint64_t layer;
4668
4669         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4670         if (!ret) {
4671                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4672                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4673                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4674                 if (!(item_flags & layer))
4675                         return rte_flow_error_set(error, EINVAL,
4676                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4677                                                   NULL, "no TCP item in"
4678                                                   " pattern");
4679                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
4680                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
4681                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
4682                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
4683                         return rte_flow_error_set(error, EINVAL,
4684                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4685                                                   NULL,
4686                                                   "cannot decrease and increase"
4687                                                   " TCP acknowledgment number"
4688                                                   " at the same time");
4689         }
4690         return ret;
4691 }
4692
4693 /**
4694  * Validate the modify-header TTL actions.
4695  *
4696  * @param[in] action_flags
4697  *   Holds the actions detected until now.
4698  * @param[in] action
4699  *   Pointer to the modify action.
4700  * @param[in] item_flags
4701  *   Holds the items detected.
4702  * @param[out] error
4703  *   Pointer to error structure.
4704  *
4705  * @return
4706  *   0 on success, a negative errno value otherwise and rte_errno is set.
4707  */
4708 static int
4709 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
4710                                    const struct rte_flow_action *action,
4711                                    const uint64_t item_flags,
4712                                    struct rte_flow_error *error)
4713 {
4714         int ret = 0;
4715         uint64_t layer;
4716
4717         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4718         if (!ret) {
4719                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4720                                  MLX5_FLOW_LAYER_INNER_L3 :
4721                                  MLX5_FLOW_LAYER_OUTER_L3;
4722                 if (!(item_flags & layer))
4723                         return rte_flow_error_set(error, EINVAL,
4724                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4725                                                   NULL,
4726                                                   "no IP protocol in pattern");
4727         }
4728         return ret;
4729 }
4730
4731 /**
4732  * Validate the generic modify field actions.
4733  * @param[in] dev
4734  *   Pointer to the rte_eth_dev structure.
4735  * @param[in] action_flags
4736  *   Holds the actions detected until now.
4737  * @param[in] action
4738  *   Pointer to the modify action.
4739  * @param[in] attr
4740  *   Pointer to the flow attributes.
4741  * @param[out] error
4742  *   Pointer to error structure.
4743  *
4744  * @return
4745  *   Number of header fields to modify (0 or more) on success,
4746  *   a negative errno value otherwise and rte_errno is set.
4747  */
4748 static int
4749 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev,
4750                                    const uint64_t action_flags,
4751                                    const struct rte_flow_action *action,
4752                                    const struct rte_flow_attr *attr,
4753                                    struct rte_flow_error *error)
4754 {
4755         int ret = 0;
4756         struct mlx5_priv *priv = dev->data->dev_private;
4757         struct mlx5_dev_config *config = &priv->config;
4758         const struct rte_flow_action_modify_field *action_modify_field =
4759                 action->conf;
4760         uint32_t dst_width = mlx5_flow_item_field_width(config,
4761                                 action_modify_field->dst.field);
4762         uint32_t src_width = mlx5_flow_item_field_width(config,
4763                                 action_modify_field->src.field);
4764
4765         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4766         if (ret)
4767                 return ret;
4768
4769         if (action_modify_field->width == 0)
4770                 return rte_flow_error_set(error, EINVAL,
4771                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4772                                 "no bits are requested to be modified");
4773         else if (action_modify_field->width > dst_width ||
4774                  action_modify_field->width > src_width)
4775                 return rte_flow_error_set(error, EINVAL,
4776                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4777                                 "cannot modify more bits than"
4778                                 " the width of a field");
4779         if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE &&
4780             action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) {
4781                 if ((action_modify_field->dst.offset +
4782                      action_modify_field->width > dst_width) ||
4783                     (action_modify_field->dst.offset % 32))
4784                         return rte_flow_error_set(error, EINVAL,
4785                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4786                                         "destination offset is too big"
4787                                         " or not aligned to 4 bytes");
4788                 if (action_modify_field->dst.level &&
4789                     action_modify_field->dst.field != RTE_FLOW_FIELD_TAG)
4790                         return rte_flow_error_set(error, ENOTSUP,
4791                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4792                                         "inner header fields modification"
4793                                         " is not supported");
4794         }
4795         if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE &&
4796             action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) {
4797                 if (!attr->transfer && !attr->group)
4798                         return rte_flow_error_set(error, ENOTSUP,
4799                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4800                                         "modify field action is not"
4801                                         " supported for group 0");
4802                 if ((action_modify_field->src.offset +
4803                      action_modify_field->width > src_width) ||
4804                     (action_modify_field->src.offset % 32))
4805                         return rte_flow_error_set(error, EINVAL,
4806                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4807                                         "source offset is too big"
4808                                         " or not aligned to 4 bytes");
4809                 if (action_modify_field->src.level &&
4810                     action_modify_field->src.field != RTE_FLOW_FIELD_TAG)
4811                         return rte_flow_error_set(error, ENOTSUP,
4812                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4813                                         "inner header fields modification"
4814                                         " is not supported");
4815         }
4816         if ((action_modify_field->dst.field ==
4817              action_modify_field->src.field) &&
4818             (action_modify_field->dst.level ==
4819              action_modify_field->src.level))
4820                 return rte_flow_error_set(error, EINVAL,
4821                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4822                                 "source and destination fields"
4823                                 " cannot be the same");
4824         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE ||
4825             action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER)
4826                 return rte_flow_error_set(error, EINVAL,
4827                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4828                                 "immediate value or a pointer to it"
4829                                 " cannot be used as a destination");
4830         if (action_modify_field->dst.field == RTE_FLOW_FIELD_START ||
4831             action_modify_field->src.field == RTE_FLOW_FIELD_START)
4832                 return rte_flow_error_set(error, ENOTSUP,
4833                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4834                                 "modifications of an arbitrary"
4835                                 " place in a packet is not supported");
4836         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE ||
4837             action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE)
4838                 return rte_flow_error_set(error, ENOTSUP,
4839                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4840                                 "modifications of the 802.1Q Tag"
4841                                 " Identifier is not supported");
4842         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI ||
4843             action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI)
4844                 return rte_flow_error_set(error, ENOTSUP,
4845                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4846                                 "modifications of the VXLAN Network"
4847                                 " Identifier is not supported");
4848         if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI ||
4849             action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI)
4850                 return rte_flow_error_set(error, ENOTSUP,
4851                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4852                                 "modifications of the GENEVE Network"
4853                                 " Identifier is not supported");
4854         if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK ||
4855             action_modify_field->src.field == RTE_FLOW_FIELD_MARK ||
4856             action_modify_field->dst.field == RTE_FLOW_FIELD_META ||
4857             action_modify_field->src.field == RTE_FLOW_FIELD_META) {
4858                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4859                     !mlx5_flow_ext_mreg_supported(dev))
4860                         return rte_flow_error_set(error, ENOTSUP,
4861                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4862                                         "cannot modify mark or metadata without"
4863                                         " extended metadata register support");
4864         }
4865         if (action_modify_field->operation != RTE_FLOW_MODIFY_SET)
4866                 return rte_flow_error_set(error, ENOTSUP,
4867                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4868                                 "add and sub operations"
4869                                 " are not supported");
4870         return (action_modify_field->width / 32) +
4871                !!(action_modify_field->width % 32);
4872 }
4873
4874 /**
4875  * Validate jump action.
4876  *
4877  * @param[in] action
4878  *   Pointer to the jump action.
4879  * @param[in] action_flags
4880  *   Holds the actions detected until now.
4881  * @param[in] attributes
4882  *   Pointer to flow attributes
4883  * @param[in] external
4884  *   Action belongs to flow rule created by request external to PMD.
4885  * @param[out] error
4886  *   Pointer to error structure.
4887  *
4888  * @return
4889  *   0 on success, a negative errno value otherwise and rte_errno is set.
4890  */
4891 static int
4892 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
4893                              const struct mlx5_flow_tunnel *tunnel,
4894                              const struct rte_flow_action *action,
4895                              uint64_t action_flags,
4896                              const struct rte_flow_attr *attributes,
4897                              bool external, struct rte_flow_error *error)
4898 {
4899         uint32_t target_group, table;
4900         int ret = 0;
4901         struct flow_grp_info grp_info = {
4902                 .external = !!external,
4903                 .transfer = !!attributes->transfer,
4904                 .fdb_def_rule = 1,
4905                 .std_tbl_fix = 0
4906         };
4907         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4908                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4909                 return rte_flow_error_set(error, EINVAL,
4910                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4911                                           "can't have 2 fate actions in"
4912                                           " same flow");
4913         if (!action->conf)
4914                 return rte_flow_error_set(error, EINVAL,
4915                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4916                                           NULL, "action configuration not set");
4917         target_group =
4918                 ((const struct rte_flow_action_jump *)action->conf)->group;
4919         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
4920                                        &grp_info, error);
4921         if (ret)
4922                 return ret;
4923         if (attributes->group == target_group &&
4924             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
4925                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
4926                 return rte_flow_error_set(error, EINVAL,
4927                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4928                                           "target group must be other than"
4929                                           " the current flow group");
4930         return 0;
4931 }
4932
4933 /*
4934  * Validate the port_id action.
4935  *
4936  * @param[in] dev
4937  *   Pointer to rte_eth_dev structure.
4938  * @param[in] action_flags
4939  *   Bit-fields that holds the actions detected until now.
4940  * @param[in] action
4941  *   Port_id RTE action structure.
4942  * @param[in] attr
4943  *   Attributes of flow that includes this action.
4944  * @param[out] error
4945  *   Pointer to error structure.
4946  *
4947  * @return
4948  *   0 on success, a negative errno value otherwise and rte_errno is set.
4949  */
4950 static int
4951 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
4952                                 uint64_t action_flags,
4953                                 const struct rte_flow_action *action,
4954                                 const struct rte_flow_attr *attr,
4955                                 struct rte_flow_error *error)
4956 {
4957         const struct rte_flow_action_port_id *port_id;
4958         struct mlx5_priv *act_priv;
4959         struct mlx5_priv *dev_priv;
4960         uint16_t port;
4961
4962         if (!attr->transfer)
4963                 return rte_flow_error_set(error, ENOTSUP,
4964                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4965                                           NULL,
4966                                           "port id action is valid in transfer"
4967                                           " mode only");
4968         if (!action || !action->conf)
4969                 return rte_flow_error_set(error, ENOTSUP,
4970                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4971                                           NULL,
4972                                           "port id action parameters must be"
4973                                           " specified");
4974         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4975                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4976                 return rte_flow_error_set(error, EINVAL,
4977                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4978                                           "can have only one fate actions in"
4979                                           " a flow");
4980         dev_priv = mlx5_dev_to_eswitch_info(dev);
4981         if (!dev_priv)
4982                 return rte_flow_error_set(error, rte_errno,
4983                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4984                                           NULL,
4985                                           "failed to obtain E-Switch info");
4986         port_id = action->conf;
4987         port = port_id->original ? dev->data->port_id : port_id->id;
4988         act_priv = mlx5_port_to_eswitch_info(port, false);
4989         if (!act_priv)
4990                 return rte_flow_error_set
4991                                 (error, rte_errno,
4992                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
4993                                  "failed to obtain E-Switch port id for port");
4994         if (act_priv->domain_id != dev_priv->domain_id)
4995                 return rte_flow_error_set
4996                                 (error, EINVAL,
4997                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4998                                  "port does not belong to"
4999                                  " E-Switch being configured");
5000         return 0;
5001 }
5002
5003 /**
5004  * Get the maximum number of modify header actions.
5005  *
5006  * @param dev
5007  *   Pointer to rte_eth_dev structure.
5008  * @param flags
5009  *   Flags bits to check if root level.
5010  *
5011  * @return
5012  *   Max number of modify header actions device can support.
5013  */
5014 static inline unsigned int
5015 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
5016                               uint64_t flags)
5017 {
5018         /*
5019          * There's no way to directly query the max capacity from FW.
5020          * The maximal value on root table should be assumed to be supported.
5021          */
5022         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
5023                 return MLX5_MAX_MODIFY_NUM;
5024         else
5025                 return MLX5_ROOT_TBL_MODIFY_NUM;
5026 }
5027
5028 /**
5029  * Validate the meter action.
5030  *
5031  * @param[in] dev
5032  *   Pointer to rte_eth_dev structure.
5033  * @param[in] action_flags
5034  *   Bit-fields that holds the actions detected until now.
5035  * @param[in] action
5036  *   Pointer to the meter action.
5037  * @param[in] attr
5038  *   Attributes of flow that includes this action.
5039  * @param[in] port_id_item
5040  *   Pointer to item indicating port id.
5041  * @param[out] error
5042  *   Pointer to error structure.
5043  *
5044  * @return
5045  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5046  */
5047 static int
5048 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
5049                                 uint64_t action_flags,
5050                                 const struct rte_flow_action *action,
5051                                 const struct rte_flow_attr *attr,
5052                                 const struct rte_flow_item *port_id_item,
5053                                 bool *def_policy,
5054                                 struct rte_flow_error *error)
5055 {
5056         struct mlx5_priv *priv = dev->data->dev_private;
5057         const struct rte_flow_action_meter *am = action->conf;
5058         struct mlx5_flow_meter_info *fm;
5059         struct mlx5_flow_meter_policy *mtr_policy;
5060         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
5061
5062         if (!am)
5063                 return rte_flow_error_set(error, EINVAL,
5064                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5065                                           "meter action conf is NULL");
5066
5067         if (action_flags & MLX5_FLOW_ACTION_METER)
5068                 return rte_flow_error_set(error, ENOTSUP,
5069                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5070                                           "meter chaining not support");
5071         if (action_flags & MLX5_FLOW_ACTION_JUMP)
5072                 return rte_flow_error_set(error, ENOTSUP,
5073                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5074                                           "meter with jump not support");
5075         if (!priv->mtr_en)
5076                 return rte_flow_error_set(error, ENOTSUP,
5077                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5078                                           NULL,
5079                                           "meter action not supported");
5080         fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL);
5081         if (!fm)
5082                 return rte_flow_error_set(error, EINVAL,
5083                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5084                                           "Meter not found");
5085         /* aso meter can always be shared by different domains */
5086         if (fm->ref_cnt && !priv->sh->meter_aso_en &&
5087             !(fm->transfer == attr->transfer ||
5088               (!fm->ingress && !attr->ingress && attr->egress) ||
5089               (!fm->egress && !attr->egress && attr->ingress)))
5090                 return rte_flow_error_set(error, EINVAL,
5091                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5092                         "Flow attributes domain are either invalid "
5093                         "or have a domain conflict with current "
5094                         "meter attributes");
5095         if (fm->def_policy) {
5096                 if (!((attr->transfer &&
5097                         mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) ||
5098                         (attr->egress &&
5099                         mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) ||
5100                         (attr->ingress &&
5101                         mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS])))
5102                         return rte_flow_error_set(error, EINVAL,
5103                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5104                                           "Flow attributes domain "
5105                                           "have a conflict with current "
5106                                           "meter domain attributes");
5107                 *def_policy = true;
5108         } else {
5109                 mtr_policy = mlx5_flow_meter_policy_find(dev,
5110                                                 fm->policy_id, NULL);
5111                 if (!mtr_policy)
5112                         return rte_flow_error_set(error, EINVAL,
5113                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5114                                           "Invalid policy id for meter ");
5115                 if (!((attr->transfer && mtr_policy->transfer) ||
5116                         (attr->egress && mtr_policy->egress) ||
5117                         (attr->ingress && mtr_policy->ingress)))
5118                         return rte_flow_error_set(error, EINVAL,
5119                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5120                                           "Flow attributes domain "
5121                                           "have a conflict with current "
5122                                           "meter domain attributes");
5123                 if (attr->transfer && mtr_policy->dev) {
5124                         /**
5125                          * When policy has fate action of port_id,
5126                          * the flow should have the same src port as policy.
5127                          */
5128                         struct mlx5_priv *policy_port_priv =
5129                                         mtr_policy->dev->data->dev_private;
5130                         int32_t flow_src_port = priv->representor_id;
5131
5132                         if (port_id_item) {
5133                                 const struct rte_flow_item_port_id *spec =
5134                                                         port_id_item->spec;
5135                                 struct mlx5_priv *port_priv =
5136                                         mlx5_port_to_eswitch_info(spec->id,
5137                                                                   false);
5138                                 if (!port_priv)
5139                                         return rte_flow_error_set(error,
5140                                                 rte_errno,
5141                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5142                                                 spec,
5143                                                 "Failed to get port info.");
5144                                 flow_src_port = port_priv->representor_id;
5145                         }
5146                         if (flow_src_port != policy_port_priv->representor_id)
5147                                 return rte_flow_error_set(error,
5148                                                 rte_errno,
5149                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5150                                                 NULL,
5151                                                 "Flow and meter policy "
5152                                                 "have different src port.");
5153                 }
5154                 *def_policy = false;
5155         }
5156         return 0;
5157 }
5158
5159 /**
5160  * Validate the age action.
5161  *
5162  * @param[in] action_flags
5163  *   Holds the actions detected until now.
5164  * @param[in] action
5165  *   Pointer to the age action.
5166  * @param[in] dev
5167  *   Pointer to the Ethernet device structure.
5168  * @param[out] error
5169  *   Pointer to error structure.
5170  *
5171  * @return
5172  *   0 on success, a negative errno value otherwise and rte_errno is set.
5173  */
5174 static int
5175 flow_dv_validate_action_age(uint64_t action_flags,
5176                             const struct rte_flow_action *action,
5177                             struct rte_eth_dev *dev,
5178                             struct rte_flow_error *error)
5179 {
5180         struct mlx5_priv *priv = dev->data->dev_private;
5181         const struct rte_flow_action_age *age = action->conf;
5182
5183         if (!priv->config.devx || (priv->sh->cmng.counter_fallback &&
5184             !priv->sh->aso_age_mng))
5185                 return rte_flow_error_set(error, ENOTSUP,
5186                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5187                                           NULL,
5188                                           "age action not supported");
5189         if (!(action->conf))
5190                 return rte_flow_error_set(error, EINVAL,
5191                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5192                                           "configuration cannot be null");
5193         if (!(age->timeout))
5194                 return rte_flow_error_set(error, EINVAL,
5195                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5196                                           "invalid timeout value 0");
5197         if (action_flags & MLX5_FLOW_ACTION_AGE)
5198                 return rte_flow_error_set(error, EINVAL,
5199                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5200                                           "duplicate age actions set");
5201         return 0;
5202 }
5203
5204 /**
5205  * Validate the modify-header IPv4 DSCP actions.
5206  *
5207  * @param[in] action_flags
5208  *   Holds the actions detected until now.
5209  * @param[in] action
5210  *   Pointer to the modify action.
5211  * @param[in] item_flags
5212  *   Holds the items detected.
5213  * @param[out] error
5214  *   Pointer to error structure.
5215  *
5216  * @return
5217  *   0 on success, a negative errno value otherwise and rte_errno is set.
5218  */
5219 static int
5220 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
5221                                          const struct rte_flow_action *action,
5222                                          const uint64_t item_flags,
5223                                          struct rte_flow_error *error)
5224 {
5225         int ret = 0;
5226
5227         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5228         if (!ret) {
5229                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
5230                         return rte_flow_error_set(error, EINVAL,
5231                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5232                                                   NULL,
5233                                                   "no ipv4 item in pattern");
5234         }
5235         return ret;
5236 }
5237
5238 /**
5239  * Validate the modify-header IPv6 DSCP actions.
5240  *
5241  * @param[in] action_flags
5242  *   Holds the actions detected until now.
5243  * @param[in] action
5244  *   Pointer to the modify action.
5245  * @param[in] item_flags
5246  *   Holds the items detected.
5247  * @param[out] error
5248  *   Pointer to error structure.
5249  *
5250  * @return
5251  *   0 on success, a negative errno value otherwise and rte_errno is set.
5252  */
5253 static int
5254 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
5255                                          const struct rte_flow_action *action,
5256                                          const uint64_t item_flags,
5257                                          struct rte_flow_error *error)
5258 {
5259         int ret = 0;
5260
5261         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5262         if (!ret) {
5263                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
5264                         return rte_flow_error_set(error, EINVAL,
5265                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5266                                                   NULL,
5267                                                   "no ipv6 item in pattern");
5268         }
5269         return ret;
5270 }
5271
5272 /**
5273  * Match modify-header resource.
5274  *
5275  * @param list
5276  *   Pointer to the hash list.
5277  * @param entry
5278  *   Pointer to exist resource entry object.
5279  * @param key
5280  *   Key of the new entry.
5281  * @param ctx
5282  *   Pointer to new modify-header resource.
5283  *
5284  * @return
5285  *   0 on matching, non-zero otherwise.
5286  */
5287 int
5288 flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
5289                         struct mlx5_hlist_entry *entry,
5290                         uint64_t key __rte_unused, void *cb_ctx)
5291 {
5292         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5293         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5294         struct mlx5_flow_dv_modify_hdr_resource *resource =
5295                         container_of(entry, typeof(*resource), entry);
5296         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5297
5298         key_len += ref->actions_num * sizeof(ref->actions[0]);
5299         return ref->actions_num != resource->actions_num ||
5300                memcmp(&ref->ft_type, &resource->ft_type, key_len);
5301 }
5302
5303 struct mlx5_hlist_entry *
5304 flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
5305                          void *cb_ctx)
5306 {
5307         struct mlx5_dev_ctx_shared *sh = list->ctx;
5308         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5309         struct mlx5dv_dr_domain *ns;
5310         struct mlx5_flow_dv_modify_hdr_resource *entry;
5311         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5312         int ret;
5313         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5314         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5315
5316         entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0,
5317                             SOCKET_ID_ANY);
5318         if (!entry) {
5319                 rte_flow_error_set(ctx->error, ENOMEM,
5320                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5321                                    "cannot allocate resource memory");
5322                 return NULL;
5323         }
5324         rte_memcpy(&entry->ft_type,
5325                    RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
5326                    key_len + data_len);
5327         if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
5328                 ns = sh->fdb_domain;
5329         else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
5330                 ns = sh->tx_domain;
5331         else
5332                 ns = sh->rx_domain;
5333         ret = mlx5_flow_os_create_flow_action_modify_header
5334                                         (sh->ctx, ns, entry,
5335                                          data_len, &entry->action);
5336         if (ret) {
5337                 mlx5_free(entry);
5338                 rte_flow_error_set(ctx->error, ENOMEM,
5339                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5340                                    NULL, "cannot create modification action");
5341                 return NULL;
5342         }
5343         return &entry->entry;
5344 }
5345
5346 /**
5347  * Validate the sample action.
5348  *
5349  * @param[in, out] action_flags
5350  *   Holds the actions detected until now.
5351  * @param[in] action
5352  *   Pointer to the sample action.
5353  * @param[in] dev
5354  *   Pointer to the Ethernet device structure.
5355  * @param[in] attr
5356  *   Attributes of flow that includes this action.
5357  * @param[in] item_flags
5358  *   Holds the items detected.
5359  * @param[in] rss
5360  *   Pointer to the RSS action.
5361  * @param[out] sample_rss
5362  *   Pointer to the RSS action in sample action list.
5363  * @param[out] count
5364  *   Pointer to the COUNT action in sample action list.
5365  * @param[out] fdb_mirror_limit
5366  *   Pointer to the FDB mirror limitation flag.
5367  * @param[out] error
5368  *   Pointer to error structure.
5369  *
5370  * @return
5371  *   0 on success, a negative errno value otherwise and rte_errno is set.
5372  */
5373 static int
5374 flow_dv_validate_action_sample(uint64_t *action_flags,
5375                                const struct rte_flow_action *action,
5376                                struct rte_eth_dev *dev,
5377                                const struct rte_flow_attr *attr,
5378                                uint64_t item_flags,
5379                                const struct rte_flow_action_rss *rss,
5380                                const struct rte_flow_action_rss **sample_rss,
5381                                const struct rte_flow_action_count **count,
5382                                int *fdb_mirror_limit,
5383                                struct rte_flow_error *error)
5384 {
5385         struct mlx5_priv *priv = dev->data->dev_private;
5386         struct mlx5_dev_config *dev_conf = &priv->config;
5387         const struct rte_flow_action_sample *sample = action->conf;
5388         const struct rte_flow_action *act;
5389         uint64_t sub_action_flags = 0;
5390         uint16_t queue_index = 0xFFFF;
5391         int actions_n = 0;
5392         int ret;
5393
5394         if (!sample)
5395                 return rte_flow_error_set(error, EINVAL,
5396                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5397                                           "configuration cannot be NULL");
5398         if (sample->ratio == 0)
5399                 return rte_flow_error_set(error, EINVAL,
5400                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5401                                           "ratio value starts from 1");
5402         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
5403                 return rte_flow_error_set(error, ENOTSUP,
5404                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5405                                           NULL,
5406                                           "sample action not supported");
5407         if (*action_flags & MLX5_FLOW_ACTION_SAMPLE)
5408                 return rte_flow_error_set(error, EINVAL,
5409                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5410                                           "Multiple sample actions not "
5411                                           "supported");
5412         if (*action_flags & MLX5_FLOW_ACTION_METER)
5413                 return rte_flow_error_set(error, EINVAL,
5414                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5415                                           "wrong action order, meter should "
5416                                           "be after sample action");
5417         if (*action_flags & MLX5_FLOW_ACTION_JUMP)
5418                 return rte_flow_error_set(error, EINVAL,
5419                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5420                                           "wrong action order, jump should "
5421                                           "be after sample action");
5422         act = sample->actions;
5423         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
5424                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5425                         return rte_flow_error_set(error, ENOTSUP,
5426                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5427                                                   act, "too many actions");
5428                 switch (act->type) {
5429                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5430                         ret = mlx5_flow_validate_action_queue(act,
5431                                                               sub_action_flags,
5432                                                               dev,
5433                                                               attr, error);
5434                         if (ret < 0)
5435                                 return ret;
5436                         queue_index = ((const struct rte_flow_action_queue *)
5437                                                         (act->conf))->index;
5438                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
5439                         ++actions_n;
5440                         break;
5441                 case RTE_FLOW_ACTION_TYPE_RSS:
5442                         *sample_rss = act->conf;
5443                         ret = mlx5_flow_validate_action_rss(act,
5444                                                             sub_action_flags,
5445                                                             dev, attr,
5446                                                             item_flags,
5447                                                             error);
5448                         if (ret < 0)
5449                                 return ret;
5450                         if (rss && *sample_rss &&
5451                             ((*sample_rss)->level != rss->level ||
5452                             (*sample_rss)->types != rss->types))
5453                                 return rte_flow_error_set(error, ENOTSUP,
5454                                         RTE_FLOW_ERROR_TYPE_ACTION,
5455                                         NULL,
5456                                         "Can't use the different RSS types "
5457                                         "or level in the same flow");
5458                         if (*sample_rss != NULL && (*sample_rss)->queue_num)
5459                                 queue_index = (*sample_rss)->queue[0];
5460                         sub_action_flags |= MLX5_FLOW_ACTION_RSS;
5461                         ++actions_n;
5462                         break;
5463                 case RTE_FLOW_ACTION_TYPE_MARK:
5464                         ret = flow_dv_validate_action_mark(dev, act,
5465                                                            sub_action_flags,
5466                                                            attr, error);
5467                         if (ret < 0)
5468                                 return ret;
5469                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
5470                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
5471                                                 MLX5_FLOW_ACTION_MARK_EXT;
5472                         else
5473                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
5474                         ++actions_n;
5475                         break;
5476                 case RTE_FLOW_ACTION_TYPE_COUNT:
5477                         ret = flow_dv_validate_action_count
5478                                 (dev, is_shared_action_count(act),
5479                                  *action_flags | sub_action_flags,
5480                                  error);
5481                         if (ret < 0)
5482                                 return ret;
5483                         *count = act->conf;
5484                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
5485                         *action_flags |= MLX5_FLOW_ACTION_COUNT;
5486                         ++actions_n;
5487                         break;
5488                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5489                         ret = flow_dv_validate_action_port_id(dev,
5490                                                               sub_action_flags,
5491                                                               act,
5492                                                               attr,
5493                                                               error);
5494                         if (ret)
5495                                 return ret;
5496                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5497                         ++actions_n;
5498                         break;
5499                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5500                         ret = flow_dv_validate_action_raw_encap_decap
5501                                 (dev, NULL, act->conf, attr, &sub_action_flags,
5502                                  &actions_n, action, item_flags, error);
5503                         if (ret < 0)
5504                                 return ret;
5505                         ++actions_n;
5506                         break;
5507                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5508                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5509                         ret = flow_dv_validate_action_l2_encap(dev,
5510                                                                sub_action_flags,
5511                                                                act, attr,
5512                                                                error);
5513                         if (ret < 0)
5514                                 return ret;
5515                         sub_action_flags |= MLX5_FLOW_ACTION_ENCAP;
5516                         ++actions_n;
5517                         break;
5518                 default:
5519                         return rte_flow_error_set(error, ENOTSUP,
5520                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5521                                                   NULL,
5522                                                   "Doesn't support optional "
5523                                                   "action");
5524                 }
5525         }
5526         if (attr->ingress && !attr->transfer) {
5527                 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE |
5528                                           MLX5_FLOW_ACTION_RSS)))
5529                         return rte_flow_error_set(error, EINVAL,
5530                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5531                                                   NULL,
5532                                                   "Ingress must has a dest "
5533                                                   "QUEUE for Sample");
5534         } else if (attr->egress && !attr->transfer) {
5535                 return rte_flow_error_set(error, ENOTSUP,
5536                                           RTE_FLOW_ERROR_TYPE_ACTION,
5537                                           NULL,
5538                                           "Sample Only support Ingress "
5539                                           "or E-Switch");
5540         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
5541                 MLX5_ASSERT(attr->transfer);
5542                 if (sample->ratio > 1)
5543                         return rte_flow_error_set(error, ENOTSUP,
5544                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5545                                                   NULL,
5546                                                   "E-Switch doesn't support "
5547                                                   "any optional action "
5548                                                   "for sampling");
5549                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
5550                         return rte_flow_error_set(error, ENOTSUP,
5551                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5552                                                   NULL,
5553                                                   "unsupported action QUEUE");
5554                 if (sub_action_flags & MLX5_FLOW_ACTION_RSS)
5555                         return rte_flow_error_set(error, ENOTSUP,
5556                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5557                                                   NULL,
5558                                                   "unsupported action QUEUE");
5559                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
5560                         return rte_flow_error_set(error, EINVAL,
5561                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5562                                                   NULL,
5563                                                   "E-Switch must has a dest "
5564                                                   "port for mirroring");
5565                 if (!priv->config.hca_attr.reg_c_preserve &&
5566                      priv->representor_id != UINT16_MAX)
5567                         *fdb_mirror_limit = 1;
5568         }
5569         /* Continue validation for Xcap actions.*/
5570         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
5571             (queue_index == 0xFFFF ||
5572              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5573                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5574                      MLX5_FLOW_XCAP_ACTIONS)
5575                         return rte_flow_error_set(error, ENOTSUP,
5576                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5577                                                   NULL, "encap and decap "
5578                                                   "combination aren't "
5579                                                   "supported");
5580                 if (!attr->transfer && attr->ingress && (sub_action_flags &
5581                                                         MLX5_FLOW_ACTION_ENCAP))
5582                         return rte_flow_error_set(error, ENOTSUP,
5583                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5584                                                   NULL, "encap is not supported"
5585                                                   " for ingress traffic");
5586         }
5587         return 0;
5588 }
5589
5590 /**
5591  * Find existing modify-header resource or create and register a new one.
5592  *
5593  * @param dev[in, out]
5594  *   Pointer to rte_eth_dev structure.
5595  * @param[in, out] resource
5596  *   Pointer to modify-header resource.
5597  * @parm[in, out] dev_flow
5598  *   Pointer to the dev_flow.
5599  * @param[out] error
5600  *   pointer to error structure.
5601  *
5602  * @return
5603  *   0 on success otherwise -errno and errno is set.
5604  */
5605 static int
5606 flow_dv_modify_hdr_resource_register
5607                         (struct rte_eth_dev *dev,
5608                          struct mlx5_flow_dv_modify_hdr_resource *resource,
5609                          struct mlx5_flow *dev_flow,
5610                          struct rte_flow_error *error)
5611 {
5612         struct mlx5_priv *priv = dev->data->dev_private;
5613         struct mlx5_dev_ctx_shared *sh = priv->sh;
5614         uint32_t key_len = sizeof(*resource) -
5615                            offsetof(typeof(*resource), ft_type) +
5616                            resource->actions_num * sizeof(resource->actions[0]);
5617         struct mlx5_hlist_entry *entry;
5618         struct mlx5_flow_cb_ctx ctx = {
5619                 .error = error,
5620                 .data = resource,
5621         };
5622         uint64_t key64;
5623
5624         resource->flags = dev_flow->dv.group ? 0 :
5625                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5626         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
5627                                     resource->flags))
5628                 return rte_flow_error_set(error, EOVERFLOW,
5629                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5630                                           "too many modify header items");
5631         key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
5632         entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
5633         if (!entry)
5634                 return -rte_errno;
5635         resource = container_of(entry, typeof(*resource), entry);
5636         dev_flow->handle->dvh.modify_hdr = resource;
5637         return 0;
5638 }
5639
5640 /**
5641  * Get DV flow counter by index.
5642  *
5643  * @param[in] dev
5644  *   Pointer to the Ethernet device structure.
5645  * @param[in] idx
5646  *   mlx5 flow counter index in the container.
5647  * @param[out] ppool
5648  *   mlx5 flow counter pool in the container.
5649  *
5650  * @return
5651  *   Pointer to the counter, NULL otherwise.
5652  */
5653 static struct mlx5_flow_counter *
5654 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
5655                            uint32_t idx,
5656                            struct mlx5_flow_counter_pool **ppool)
5657 {
5658         struct mlx5_priv *priv = dev->data->dev_private;
5659         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5660         struct mlx5_flow_counter_pool *pool;
5661
5662         /* Decrease to original index and clear shared bit. */
5663         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
5664         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
5665         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
5666         MLX5_ASSERT(pool);
5667         if (ppool)
5668                 *ppool = pool;
5669         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
5670 }
5671
5672 /**
5673  * Check the devx counter belongs to the pool.
5674  *
5675  * @param[in] pool
5676  *   Pointer to the counter pool.
5677  * @param[in] id
5678  *   The counter devx ID.
5679  *
5680  * @return
5681  *   True if counter belongs to the pool, false otherwise.
5682  */
5683 static bool
5684 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
5685 {
5686         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
5687                    MLX5_COUNTERS_PER_POOL;
5688
5689         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
5690                 return true;
5691         return false;
5692 }
5693
5694 /**
5695  * Get a pool by devx counter ID.
5696  *
5697  * @param[in] cmng
5698  *   Pointer to the counter management.
5699  * @param[in] id
5700  *   The counter devx ID.
5701  *
5702  * @return
5703  *   The counter pool pointer if exists, NULL otherwise,
5704  */
5705 static struct mlx5_flow_counter_pool *
5706 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
5707 {
5708         uint32_t i;
5709         struct mlx5_flow_counter_pool *pool = NULL;
5710
5711         rte_spinlock_lock(&cmng->pool_update_sl);
5712         /* Check last used pool. */
5713         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
5714             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
5715                 pool = cmng->pools[cmng->last_pool_idx];
5716                 goto out;
5717         }
5718         /* ID out of range means no suitable pool in the container. */
5719         if (id > cmng->max_id || id < cmng->min_id)
5720                 goto out;
5721         /*
5722          * Find the pool from the end of the container, since mostly counter
5723          * ID is sequence increasing, and the last pool should be the needed
5724          * one.
5725          */
5726         i = cmng->n_valid;
5727         while (i--) {
5728                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
5729
5730                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
5731                         pool = pool_tmp;
5732                         break;
5733                 }
5734         }
5735 out:
5736         rte_spinlock_unlock(&cmng->pool_update_sl);
5737         return pool;
5738 }
5739
5740 /**
5741  * Resize a counter container.
5742  *
5743  * @param[in] dev
5744  *   Pointer to the Ethernet device structure.
5745  *
5746  * @return
5747  *   0 on success, otherwise negative errno value and rte_errno is set.
5748  */
5749 static int
5750 flow_dv_container_resize(struct rte_eth_dev *dev)
5751 {
5752         struct mlx5_priv *priv = dev->data->dev_private;
5753         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5754         void *old_pools = cmng->pools;
5755         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
5756         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
5757         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
5758
5759         if (!pools) {
5760                 rte_errno = ENOMEM;
5761                 return -ENOMEM;
5762         }
5763         if (old_pools)
5764                 memcpy(pools, old_pools, cmng->n *
5765                                        sizeof(struct mlx5_flow_counter_pool *));
5766         cmng->n = resize;
5767         cmng->pools = pools;
5768         if (old_pools)
5769                 mlx5_free(old_pools);
5770         return 0;
5771 }
5772
5773 /**
5774  * Query a devx flow counter.
5775  *
5776  * @param[in] dev
5777  *   Pointer to the Ethernet device structure.
5778  * @param[in] counter
5779  *   Index to the flow counter.
5780  * @param[out] pkts
5781  *   The statistics value of packets.
5782  * @param[out] bytes
5783  *   The statistics value of bytes.
5784  *
5785  * @return
5786  *   0 on success, otherwise a negative errno value and rte_errno is set.
5787  */
5788 static inline int
5789 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
5790                      uint64_t *bytes)
5791 {
5792         struct mlx5_priv *priv = dev->data->dev_private;
5793         struct mlx5_flow_counter_pool *pool = NULL;
5794         struct mlx5_flow_counter *cnt;
5795         int offset;
5796
5797         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5798         MLX5_ASSERT(pool);
5799         if (priv->sh->cmng.counter_fallback)
5800                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
5801                                         0, pkts, bytes, 0, NULL, NULL, 0);
5802         rte_spinlock_lock(&pool->sl);
5803         if (!pool->raw) {
5804                 *pkts = 0;
5805                 *bytes = 0;
5806         } else {
5807                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
5808                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
5809                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
5810         }
5811         rte_spinlock_unlock(&pool->sl);
5812         return 0;
5813 }
5814
5815 /**
5816  * Create and initialize a new counter pool.
5817  *
5818  * @param[in] dev
5819  *   Pointer to the Ethernet device structure.
5820  * @param[out] dcs
5821  *   The devX counter handle.
5822  * @param[in] age
5823  *   Whether the pool is for counter that was allocated for aging.
5824  * @param[in/out] cont_cur
5825  *   Pointer to the container pointer, it will be update in pool resize.
5826  *
5827  * @return
5828  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
5829  */
5830 static struct mlx5_flow_counter_pool *
5831 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
5832                     uint32_t age)
5833 {
5834         struct mlx5_priv *priv = dev->data->dev_private;
5835         struct mlx5_flow_counter_pool *pool;
5836         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5837         bool fallback = priv->sh->cmng.counter_fallback;
5838         uint32_t size = sizeof(*pool);
5839
5840         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
5841         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
5842         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
5843         if (!pool) {
5844                 rte_errno = ENOMEM;
5845                 return NULL;
5846         }
5847         pool->raw = NULL;
5848         pool->is_aged = !!age;
5849         pool->query_gen = 0;
5850         pool->min_dcs = dcs;
5851         rte_spinlock_init(&pool->sl);
5852         rte_spinlock_init(&pool->csl);
5853         TAILQ_INIT(&pool->counters[0]);
5854         TAILQ_INIT(&pool->counters[1]);
5855         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
5856         rte_spinlock_lock(&cmng->pool_update_sl);
5857         pool->index = cmng->n_valid;
5858         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
5859                 mlx5_free(pool);
5860                 rte_spinlock_unlock(&cmng->pool_update_sl);
5861                 return NULL;
5862         }
5863         cmng->pools[pool->index] = pool;
5864         cmng->n_valid++;
5865         if (unlikely(fallback)) {
5866                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
5867
5868                 if (base < cmng->min_id)
5869                         cmng->min_id = base;
5870                 if (base > cmng->max_id)
5871                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
5872                 cmng->last_pool_idx = pool->index;
5873         }
5874         rte_spinlock_unlock(&cmng->pool_update_sl);
5875         return pool;
5876 }
5877
5878 /**
5879  * Prepare a new counter and/or a new counter pool.
5880  *
5881  * @param[in] dev
5882  *   Pointer to the Ethernet device structure.
5883  * @param[out] cnt_free
5884  *   Where to put the pointer of a new counter.
5885  * @param[in] age
5886  *   Whether the pool is for counter that was allocated for aging.
5887  *
5888  * @return
5889  *   The counter pool pointer and @p cnt_free is set on success,
5890  *   NULL otherwise and rte_errno is set.
5891  */
5892 static struct mlx5_flow_counter_pool *
5893 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
5894                              struct mlx5_flow_counter **cnt_free,
5895                              uint32_t age)
5896 {
5897         struct mlx5_priv *priv = dev->data->dev_private;
5898         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5899         struct mlx5_flow_counter_pool *pool;
5900         struct mlx5_counters tmp_tq;
5901         struct mlx5_devx_obj *dcs = NULL;
5902         struct mlx5_flow_counter *cnt;
5903         enum mlx5_counter_type cnt_type =
5904                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
5905         bool fallback = priv->sh->cmng.counter_fallback;
5906         uint32_t i;
5907
5908         if (fallback) {
5909                 /* bulk_bitmap must be 0 for single counter allocation. */
5910                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
5911                 if (!dcs)
5912                         return NULL;
5913                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
5914                 if (!pool) {
5915                         pool = flow_dv_pool_create(dev, dcs, age);
5916                         if (!pool) {
5917                                 mlx5_devx_cmd_destroy(dcs);
5918                                 return NULL;
5919                         }
5920                 }
5921                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
5922                 cnt = MLX5_POOL_GET_CNT(pool, i);
5923                 cnt->pool = pool;
5924                 cnt->dcs_when_free = dcs;
5925                 *cnt_free = cnt;
5926                 return pool;
5927         }
5928         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
5929         if (!dcs) {
5930                 rte_errno = ENODATA;
5931                 return NULL;
5932         }
5933         pool = flow_dv_pool_create(dev, dcs, age);
5934         if (!pool) {
5935                 mlx5_devx_cmd_destroy(dcs);
5936                 return NULL;
5937         }
5938         TAILQ_INIT(&tmp_tq);
5939         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
5940                 cnt = MLX5_POOL_GET_CNT(pool, i);
5941                 cnt->pool = pool;
5942                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
5943         }
5944         rte_spinlock_lock(&cmng->csl[cnt_type]);
5945         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
5946         rte_spinlock_unlock(&cmng->csl[cnt_type]);
5947         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
5948         (*cnt_free)->pool = pool;
5949         return pool;
5950 }
5951
5952 /**
5953  * Allocate a flow counter.
5954  *
5955  * @param[in] dev
5956  *   Pointer to the Ethernet device structure.
5957  * @param[in] age
5958  *   Whether the counter was allocated for aging.
5959  *
5960  * @return
5961  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5962  */
5963 static uint32_t
5964 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
5965 {
5966         struct mlx5_priv *priv = dev->data->dev_private;
5967         struct mlx5_flow_counter_pool *pool = NULL;
5968         struct mlx5_flow_counter *cnt_free = NULL;
5969         bool fallback = priv->sh->cmng.counter_fallback;
5970         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5971         enum mlx5_counter_type cnt_type =
5972                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
5973         uint32_t cnt_idx;
5974
5975         if (!priv->config.devx) {
5976                 rte_errno = ENOTSUP;
5977                 return 0;
5978         }
5979         /* Get free counters from container. */
5980         rte_spinlock_lock(&cmng->csl[cnt_type]);
5981         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
5982         if (cnt_free)
5983                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
5984         rte_spinlock_unlock(&cmng->csl[cnt_type]);
5985         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
5986                 goto err;
5987         pool = cnt_free->pool;
5988         if (fallback)
5989                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
5990         /* Create a DV counter action only in the first time usage. */
5991         if (!cnt_free->action) {
5992                 uint16_t offset;
5993                 struct mlx5_devx_obj *dcs;
5994                 int ret;
5995
5996                 if (!fallback) {
5997                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
5998                         dcs = pool->min_dcs;
5999                 } else {
6000                         offset = 0;
6001                         dcs = cnt_free->dcs_when_free;
6002                 }
6003                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
6004                                                             &cnt_free->action);
6005                 if (ret) {
6006                         rte_errno = errno;
6007                         goto err;
6008                 }
6009         }
6010         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
6011                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
6012         /* Update the counter reset values. */
6013         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
6014                                  &cnt_free->bytes))
6015                 goto err;
6016         if (!fallback && !priv->sh->cmng.query_thread_on)
6017                 /* Start the asynchronous batch query by the host thread. */
6018                 mlx5_set_query_alarm(priv->sh);
6019         /*
6020          * When the count action isn't shared (by ID), shared_info field is
6021          * used for indirect action API's refcnt.
6022          * When the counter action is not shared neither by ID nor by indirect
6023          * action API, shared info must be 1.
6024          */
6025         cnt_free->shared_info.refcnt = 1;
6026         return cnt_idx;
6027 err:
6028         if (cnt_free) {
6029                 cnt_free->pool = pool;
6030                 if (fallback)
6031                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
6032                 rte_spinlock_lock(&cmng->csl[cnt_type]);
6033                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
6034                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
6035         }
6036         return 0;
6037 }
6038
6039 /**
6040  * Allocate a shared flow counter.
6041  *
6042  * @param[in] ctx
6043  *   Pointer to the shared counter configuration.
6044  * @param[in] data
6045  *   Pointer to save the allocated counter index.
6046  *
6047  * @return
6048  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6049  */
6050
6051 static int32_t
6052 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
6053 {
6054         struct mlx5_shared_counter_conf *conf = ctx;
6055         struct rte_eth_dev *dev = conf->dev;
6056         struct mlx5_flow_counter *cnt;
6057
6058         data->dword = flow_dv_counter_alloc(dev, 0);
6059         data->dword |= MLX5_CNT_SHARED_OFFSET;
6060         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
6061         cnt->shared_info.id = conf->id;
6062         return 0;
6063 }
6064
6065 /**
6066  * Get a shared flow counter.
6067  *
6068  * @param[in] dev
6069  *   Pointer to the Ethernet device structure.
6070  * @param[in] id
6071  *   Counter identifier.
6072  *
6073  * @return
6074  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6075  */
6076 static uint32_t
6077 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
6078 {
6079         struct mlx5_priv *priv = dev->data->dev_private;
6080         struct mlx5_shared_counter_conf conf = {
6081                 .dev = dev,
6082                 .id = id,
6083         };
6084         union mlx5_l3t_data data = {
6085                 .dword = 0,
6086         };
6087
6088         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
6089                                flow_dv_counter_alloc_shared_cb, &conf);
6090         return data.dword;
6091 }
6092
6093 /**
6094  * Get age param from counter index.
6095  *
6096  * @param[in] dev
6097  *   Pointer to the Ethernet device structure.
6098  * @param[in] counter
6099  *   Index to the counter handler.
6100  *
6101  * @return
6102  *   The aging parameter specified for the counter index.
6103  */
6104 static struct mlx5_age_param*
6105 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
6106                                 uint32_t counter)
6107 {
6108         struct mlx5_flow_counter *cnt;
6109         struct mlx5_flow_counter_pool *pool = NULL;
6110
6111         flow_dv_counter_get_by_idx(dev, counter, &pool);
6112         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
6113         cnt = MLX5_POOL_GET_CNT(pool, counter);
6114         return MLX5_CNT_TO_AGE(cnt);
6115 }
6116
6117 /**
6118  * Remove a flow counter from aged counter list.
6119  *
6120  * @param[in] dev
6121  *   Pointer to the Ethernet device structure.
6122  * @param[in] counter
6123  *   Index to the counter handler.
6124  * @param[in] cnt
6125  *   Pointer to the counter handler.
6126  */
6127 static void
6128 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
6129                                 uint32_t counter, struct mlx5_flow_counter *cnt)
6130 {
6131         struct mlx5_age_info *age_info;
6132         struct mlx5_age_param *age_param;
6133         struct mlx5_priv *priv = dev->data->dev_private;
6134         uint16_t expected = AGE_CANDIDATE;
6135
6136         age_info = GET_PORT_AGE_INFO(priv);
6137         age_param = flow_dv_counter_idx_get_age(dev, counter);
6138         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
6139                                          AGE_FREE, false, __ATOMIC_RELAXED,
6140                                          __ATOMIC_RELAXED)) {
6141                 /**
6142                  * We need the lock even it is age timeout,
6143                  * since counter may still in process.
6144                  */
6145                 rte_spinlock_lock(&age_info->aged_sl);
6146                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
6147                 rte_spinlock_unlock(&age_info->aged_sl);
6148                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
6149         }
6150 }
6151
6152 /**
6153  * Release a flow counter.
6154  *
6155  * @param[in] dev
6156  *   Pointer to the Ethernet device structure.
6157  * @param[in] counter
6158  *   Index to the counter handler.
6159  */
6160 static void
6161 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter)
6162 {
6163         struct mlx5_priv *priv = dev->data->dev_private;
6164         struct mlx5_flow_counter_pool *pool = NULL;
6165         struct mlx5_flow_counter *cnt;
6166         enum mlx5_counter_type cnt_type;
6167
6168         if (!counter)
6169                 return;
6170         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6171         MLX5_ASSERT(pool);
6172         if (pool->is_aged) {
6173                 flow_dv_counter_remove_from_age(dev, counter, cnt);
6174         } else {
6175                 /*
6176                  * If the counter action is shared by ID, the l3t_clear_entry
6177                  * function reduces its references counter. If after the
6178                  * reduction the action is still referenced, the function
6179                  * returns here and does not release it.
6180                  */
6181                 if (IS_LEGACY_SHARED_CNT(counter) &&
6182                     mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
6183                                          cnt->shared_info.id))
6184                         return;
6185                 /*
6186                  * If the counter action is shared by indirect action API,
6187                  * the atomic function reduces its references counter.
6188                  * If after the reduction the action is still referenced, the
6189                  * function returns here and does not release it.
6190                  * When the counter action is not shared neither by ID nor by
6191                  * indirect action API, shared info is 1 before the reduction,
6192                  * so this condition is failed and function doesn't return here.
6193                  */
6194                 if (!IS_LEGACY_SHARED_CNT(counter) &&
6195                     __atomic_sub_fetch(&cnt->shared_info.refcnt, 1,
6196                                        __ATOMIC_RELAXED))
6197                         return;
6198         }
6199         cnt->pool = pool;
6200         /*
6201          * Put the counter back to list to be updated in none fallback mode.
6202          * Currently, we are using two list alternately, while one is in query,
6203          * add the freed counter to the other list based on the pool query_gen
6204          * value. After query finishes, add counter the list to the global
6205          * container counter list. The list changes while query starts. In
6206          * this case, lock will not be needed as query callback and release
6207          * function both operate with the different list.
6208          */
6209         if (!priv->sh->cmng.counter_fallback) {
6210                 rte_spinlock_lock(&pool->csl);
6211                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
6212                 rte_spinlock_unlock(&pool->csl);
6213         } else {
6214                 cnt->dcs_when_free = cnt->dcs_when_active;
6215                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6216                                            MLX5_COUNTER_TYPE_ORIGIN;
6217                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
6218                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
6219                                   cnt, next);
6220                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
6221         }
6222 }
6223
6224 /**
6225  * Resize a meter id container.
6226  *
6227  * @param[in] dev
6228  *   Pointer to the Ethernet device structure.
6229  *
6230  * @return
6231  *   0 on success, otherwise negative errno value and rte_errno is set.
6232  */
6233 static int
6234 flow_dv_mtr_container_resize(struct rte_eth_dev *dev)
6235 {
6236         struct mlx5_priv *priv = dev->data->dev_private;
6237         struct mlx5_aso_mtr_pools_mng *pools_mng =
6238                                 &priv->sh->mtrmng->pools_mng;
6239         void *old_pools = pools_mng->pools;
6240         uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE;
6241         uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize;
6242         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
6243
6244         if (!pools) {
6245                 rte_errno = ENOMEM;
6246                 return -ENOMEM;
6247         }
6248         if (!pools_mng->n)
6249                 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) {
6250                         mlx5_free(pools);
6251                         return -ENOMEM;
6252                 }
6253         if (old_pools)
6254                 memcpy(pools, old_pools, pools_mng->n *
6255                                        sizeof(struct mlx5_aso_mtr_pool *));
6256         pools_mng->n = resize;
6257         pools_mng->pools = pools;
6258         if (old_pools)
6259                 mlx5_free(old_pools);
6260         return 0;
6261 }
6262
6263 /**
6264  * Prepare a new meter and/or a new meter pool.
6265  *
6266  * @param[in] dev
6267  *   Pointer to the Ethernet device structure.
6268  * @param[out] mtr_free
6269  *   Where to put the pointer of a new meter.g.
6270  *
6271  * @return
6272  *   The meter pool pointer and @mtr_free is set on success,
6273  *   NULL otherwise and rte_errno is set.
6274  */
6275 static struct mlx5_aso_mtr_pool *
6276 flow_dv_mtr_pool_create(struct rte_eth_dev *dev,
6277                              struct mlx5_aso_mtr **mtr_free)
6278 {
6279         struct mlx5_priv *priv = dev->data->dev_private;
6280         struct mlx5_aso_mtr_pools_mng *pools_mng =
6281                                 &priv->sh->mtrmng->pools_mng;
6282         struct mlx5_aso_mtr_pool *pool = NULL;
6283         struct mlx5_devx_obj *dcs = NULL;
6284         uint32_t i;
6285         uint32_t log_obj_size;
6286
6287         log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
6288         dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx,
6289                         priv->sh->pdn, log_obj_size);
6290         if (!dcs) {
6291                 rte_errno = ENODATA;
6292                 return NULL;
6293         }
6294         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
6295         if (!pool) {
6296                 rte_errno = ENOMEM;
6297                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6298                 return NULL;
6299         }
6300         pool->devx_obj = dcs;
6301         pool->index = pools_mng->n_valid;
6302         if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) {
6303                 mlx5_free(pool);
6304                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6305                 return NULL;
6306         }
6307         pools_mng->pools[pool->index] = pool;
6308         pools_mng->n_valid++;
6309         for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) {
6310                 pool->mtrs[i].offset = i;
6311                 LIST_INSERT_HEAD(&pools_mng->meters,
6312                                                 &pool->mtrs[i], next);
6313         }
6314         pool->mtrs[0].offset = 0;
6315         *mtr_free = &pool->mtrs[0];
6316         return pool;
6317 }
6318
6319 /**
6320  * Release a flow meter into pool.
6321  *
6322  * @param[in] dev
6323  *   Pointer to the Ethernet device structure.
6324  * @param[in] mtr_idx
6325  *   Index to aso flow meter.
6326  */
6327 static void
6328 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx)
6329 {
6330         struct mlx5_priv *priv = dev->data->dev_private;
6331         struct mlx5_aso_mtr_pools_mng *pools_mng =
6332                                 &priv->sh->mtrmng->pools_mng;
6333         struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
6334
6335         MLX5_ASSERT(aso_mtr);
6336         rte_spinlock_lock(&pools_mng->mtrsl);
6337         memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info));
6338         aso_mtr->state = ASO_METER_FREE;
6339         LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next);
6340         rte_spinlock_unlock(&pools_mng->mtrsl);
6341 }
6342
6343 /**
6344  * Allocate a aso flow meter.
6345  *
6346  * @param[in] dev
6347  *   Pointer to the Ethernet device structure.
6348  *
6349  * @return
6350  *   Index to aso flow meter on success, 0 otherwise and rte_errno is set.
6351  */
6352 static uint32_t
6353 flow_dv_mtr_alloc(struct rte_eth_dev *dev)
6354 {
6355         struct mlx5_priv *priv = dev->data->dev_private;
6356         struct mlx5_aso_mtr *mtr_free = NULL;
6357         struct mlx5_aso_mtr_pools_mng *pools_mng =
6358                                 &priv->sh->mtrmng->pools_mng;
6359         struct mlx5_aso_mtr_pool *pool;
6360         uint32_t mtr_idx = 0;
6361
6362         if (!priv->config.devx) {
6363                 rte_errno = ENOTSUP;
6364                 return 0;
6365         }
6366         /* Allocate the flow meter memory. */
6367         /* Get free meters from management. */
6368         rte_spinlock_lock(&pools_mng->mtrsl);
6369         mtr_free = LIST_FIRST(&pools_mng->meters);
6370         if (mtr_free)
6371                 LIST_REMOVE(mtr_free, next);
6372         if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) {
6373                 rte_spinlock_unlock(&pools_mng->mtrsl);
6374                 return 0;
6375         }
6376         mtr_free->state = ASO_METER_WAIT;
6377         rte_spinlock_unlock(&pools_mng->mtrsl);
6378         pool = container_of(mtr_free,
6379                         struct mlx5_aso_mtr_pool,
6380                         mtrs[mtr_free->offset]);
6381         mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
6382         if (!mtr_free->fm.meter_action) {
6383 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
6384                 struct rte_flow_error error;
6385                 uint8_t reg_id;
6386
6387                 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
6388                 mtr_free->fm.meter_action =
6389                         mlx5_glue->dv_create_flow_action_aso
6390                                                 (priv->sh->rx_domain,
6391                                                  pool->devx_obj->obj,
6392                                                  mtr_free->offset,
6393                                                  (1 << MLX5_FLOW_COLOR_GREEN),
6394                                                  reg_id - REG_C_0);
6395 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
6396                 if (!mtr_free->fm.meter_action) {
6397                         flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
6398                         return 0;
6399                 }
6400         }
6401         return mtr_idx;
6402 }
6403
6404 /**
6405  * Verify the @p attributes will be correctly understood by the NIC and store
6406  * them in the @p flow if everything is correct.
6407  *
6408  * @param[in] dev
6409  *   Pointer to dev struct.
6410  * @param[in] attributes
6411  *   Pointer to flow attributes
6412  * @param[in] external
6413  *   This flow rule is created by request external to PMD.
6414  * @param[out] error
6415  *   Pointer to error structure.
6416  *
6417  * @return
6418  *   - 0 on success and non root table.
6419  *   - 1 on success and root table.
6420  *   - a negative errno value otherwise and rte_errno is set.
6421  */
6422 static int
6423 flow_dv_validate_attributes(struct rte_eth_dev *dev,
6424                             const struct mlx5_flow_tunnel *tunnel,
6425                             const struct rte_flow_attr *attributes,
6426                             const struct flow_grp_info *grp_info,
6427                             struct rte_flow_error *error)
6428 {
6429         struct mlx5_priv *priv = dev->data->dev_private;
6430         uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes);
6431         int ret = 0;
6432
6433 #ifndef HAVE_MLX5DV_DR
6434         RTE_SET_USED(tunnel);
6435         RTE_SET_USED(grp_info);
6436         if (attributes->group)
6437                 return rte_flow_error_set(error, ENOTSUP,
6438                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6439                                           NULL,
6440                                           "groups are not supported");
6441 #else
6442         uint32_t table = 0;
6443
6444         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
6445                                        grp_info, error);
6446         if (ret)
6447                 return ret;
6448         if (!table)
6449                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
6450 #endif
6451         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
6452             attributes->priority > lowest_priority)
6453                 return rte_flow_error_set(error, ENOTSUP,
6454                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
6455                                           NULL,
6456                                           "priority out of range");
6457         if (attributes->transfer) {
6458                 if (!priv->config.dv_esw_en)
6459                         return rte_flow_error_set
6460                                 (error, ENOTSUP,
6461                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6462                                  "E-Switch dr is not supported");
6463                 if (!(priv->representor || priv->master))
6464                         return rte_flow_error_set
6465                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6466                                  NULL, "E-Switch configuration can only be"
6467                                  " done by a master or a representor device");
6468                 if (attributes->egress)
6469                         return rte_flow_error_set
6470                                 (error, ENOTSUP,
6471                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
6472                                  "egress is not supported");
6473         }
6474         if (!(attributes->egress ^ attributes->ingress))
6475                 return rte_flow_error_set(error, ENOTSUP,
6476                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
6477                                           "must specify exactly one of "
6478                                           "ingress or egress");
6479         return ret;
6480 }
6481
6482 static uint16_t
6483 mlx5_flow_locate_proto_l3(const struct rte_flow_item **head,
6484                           const struct rte_flow_item *end)
6485 {
6486         const struct rte_flow_item *item = *head;
6487         uint16_t l3_protocol;
6488
6489         for (; item != end; item++) {
6490                 switch (item->type) {
6491                 default:
6492                         break;
6493                 case RTE_FLOW_ITEM_TYPE_IPV4:
6494                         l3_protocol = RTE_ETHER_TYPE_IPV4;
6495                         goto l3_ok;
6496                 case RTE_FLOW_ITEM_TYPE_IPV6:
6497                         l3_protocol = RTE_ETHER_TYPE_IPV6;
6498                         goto l3_ok;
6499                 case RTE_FLOW_ITEM_TYPE_ETH:
6500                         if (item->mask && item->spec) {
6501                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_eth,
6502                                                             type, item,
6503                                                             l3_protocol);
6504                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6505                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6506                                         goto l3_ok;
6507                         }
6508                         break;
6509                 case RTE_FLOW_ITEM_TYPE_VLAN:
6510                         if (item->mask && item->spec) {
6511                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_vlan,
6512                                                             inner_type, item,
6513                                                             l3_protocol);
6514                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6515                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6516                                         goto l3_ok;
6517                         }
6518                         break;
6519                 }
6520         }
6521         return 0;
6522 l3_ok:
6523         *head = item;
6524         return l3_protocol;
6525 }
6526
6527 static uint8_t
6528 mlx5_flow_locate_proto_l4(const struct rte_flow_item **head,
6529                           const struct rte_flow_item *end)
6530 {
6531         const struct rte_flow_item *item = *head;
6532         uint8_t l4_protocol;
6533
6534         for (; item != end; item++) {
6535                 switch (item->type) {
6536                 default:
6537                         break;
6538                 case RTE_FLOW_ITEM_TYPE_TCP:
6539                         l4_protocol = IPPROTO_TCP;
6540                         goto l4_ok;
6541                 case RTE_FLOW_ITEM_TYPE_UDP:
6542                         l4_protocol = IPPROTO_UDP;
6543                         goto l4_ok;
6544                 case RTE_FLOW_ITEM_TYPE_IPV4:
6545                         if (item->mask && item->spec) {
6546                                 const struct rte_flow_item_ipv4 *mask, *spec;
6547
6548                                 mask = (typeof(mask))item->mask;
6549                                 spec = (typeof(spec))item->spec;
6550                                 l4_protocol = mask->hdr.next_proto_id &
6551                                               spec->hdr.next_proto_id;
6552                                 if (l4_protocol == IPPROTO_TCP ||
6553                                     l4_protocol == IPPROTO_UDP)
6554                                         goto l4_ok;
6555                         }
6556                         break;
6557                 case RTE_FLOW_ITEM_TYPE_IPV6:
6558                         if (item->mask && item->spec) {
6559                                 const struct rte_flow_item_ipv6 *mask, *spec;
6560                                 mask = (typeof(mask))item->mask;
6561                                 spec = (typeof(spec))item->spec;
6562                                 l4_protocol = mask->hdr.proto & spec->hdr.proto;
6563                                 if (l4_protocol == IPPROTO_TCP ||
6564                                     l4_protocol == IPPROTO_UDP)
6565                                         goto l4_ok;
6566                         }
6567                         break;
6568                 }
6569         }
6570         return 0;
6571 l4_ok:
6572         *head = item;
6573         return l4_protocol;
6574 }
6575
6576 static int
6577 flow_dv_validate_item_integrity(struct rte_eth_dev *dev,
6578                                 const struct rte_flow_item *rule_items,
6579                                 const struct rte_flow_item *integrity_item,
6580                                 struct rte_flow_error *error)
6581 {
6582         struct mlx5_priv *priv = dev->data->dev_private;
6583         const struct rte_flow_item *tunnel_item, *end_item, *item = rule_items;
6584         const struct rte_flow_item_integrity *mask = (typeof(mask))
6585                                                      integrity_item->mask;
6586         const struct rte_flow_item_integrity *spec = (typeof(spec))
6587                                                      integrity_item->spec;
6588         uint32_t protocol;
6589
6590         if (!priv->config.hca_attr.pkt_integrity_match)
6591                 return rte_flow_error_set(error, ENOTSUP,
6592                                           RTE_FLOW_ERROR_TYPE_ITEM,
6593                                           integrity_item,
6594                                           "packet integrity integrity_item not supported");
6595         if (!mask)
6596                 mask = &rte_flow_item_integrity_mask;
6597         if (!mlx5_validate_integrity_item(mask))
6598                 return rte_flow_error_set(error, ENOTSUP,
6599                                           RTE_FLOW_ERROR_TYPE_ITEM,
6600                                           integrity_item,
6601                                           "unsupported integrity filter");
6602         tunnel_item = mlx5_flow_find_tunnel_item(rule_items);
6603         if (spec->level > 1) {
6604                 if (!tunnel_item)
6605                         return rte_flow_error_set(error, ENOTSUP,
6606                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6607                                                   integrity_item,
6608                                                   "missing tunnel item");
6609                 item = tunnel_item;
6610                 end_item = mlx5_find_end_item(tunnel_item);
6611         } else {
6612                 end_item = tunnel_item ? tunnel_item :
6613                            mlx5_find_end_item(integrity_item);
6614         }
6615         if (mask->l3_ok || mask->ipv4_csum_ok) {
6616                 protocol = mlx5_flow_locate_proto_l3(&item, end_item);
6617                 if (!protocol)
6618                         return rte_flow_error_set(error, EINVAL,
6619                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6620                                                   integrity_item,
6621                                                   "missing L3 protocol");
6622         }
6623         if (mask->l4_ok || mask->l4_csum_ok) {
6624                 protocol = mlx5_flow_locate_proto_l4(&item, end_item);
6625                 if (!protocol)
6626                         return rte_flow_error_set(error, EINVAL,
6627                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6628                                                   integrity_item,
6629                                                   "missing L4 protocol");
6630         }
6631         return 0;
6632 }
6633
6634 /**
6635  * Internal validation function. For validating both actions and items.
6636  *
6637  * @param[in] dev
6638  *   Pointer to the rte_eth_dev structure.
6639  * @param[in] attr
6640  *   Pointer to the flow attributes.
6641  * @param[in] items
6642  *   Pointer to the list of items.
6643  * @param[in] actions
6644  *   Pointer to the list of actions.
6645  * @param[in] external
6646  *   This flow rule is created by request external to PMD.
6647  * @param[in] hairpin
6648  *   Number of hairpin TX actions, 0 means classic flow.
6649  * @param[out] error
6650  *   Pointer to the error structure.
6651  *
6652  * @return
6653  *   0 on success, a negative errno value otherwise and rte_errno is set.
6654  */
6655 static int
6656 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
6657                  const struct rte_flow_item items[],
6658                  const struct rte_flow_action actions[],
6659                  bool external, int hairpin, struct rte_flow_error *error)
6660 {
6661         int ret;
6662         uint64_t action_flags = 0;
6663         uint64_t item_flags = 0;
6664         uint64_t last_item = 0;
6665         uint8_t next_protocol = 0xff;
6666         uint16_t ether_type = 0;
6667         int actions_n = 0;
6668         uint8_t item_ipv6_proto = 0;
6669         int fdb_mirror_limit = 0;
6670         int modify_after_mirror = 0;
6671         const struct rte_flow_item *geneve_item = NULL;
6672         const struct rte_flow_item *gre_item = NULL;
6673         const struct rte_flow_item *gtp_item = NULL;
6674         const struct rte_flow_action_raw_decap *decap;
6675         const struct rte_flow_action_raw_encap *encap;
6676         const struct rte_flow_action_rss *rss = NULL;
6677         const struct rte_flow_action_rss *sample_rss = NULL;
6678         const struct rte_flow_action_count *sample_count = NULL;
6679         const struct rte_flow_item_tcp nic_tcp_mask = {
6680                 .hdr = {
6681                         .tcp_flags = 0xFF,
6682                         .src_port = RTE_BE16(UINT16_MAX),
6683                         .dst_port = RTE_BE16(UINT16_MAX),
6684                 }
6685         };
6686         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
6687                 .hdr = {
6688                         .src_addr =
6689                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6690                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6691                         .dst_addr =
6692                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6693                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6694                         .vtc_flow = RTE_BE32(0xffffffff),
6695                         .proto = 0xff,
6696                         .hop_limits = 0xff,
6697                 },
6698                 .has_frag_ext = 1,
6699         };
6700         const struct rte_flow_item_ecpri nic_ecpri_mask = {
6701                 .hdr = {
6702                         .common = {
6703                                 .u32 =
6704                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
6705                                         .type = 0xFF,
6706                                         }).u32),
6707                         },
6708                         .dummy[0] = 0xffffffff,
6709                 },
6710         };
6711         struct mlx5_priv *priv = dev->data->dev_private;
6712         struct mlx5_dev_config *dev_conf = &priv->config;
6713         uint16_t queue_index = 0xFFFF;
6714         const struct rte_flow_item_vlan *vlan_m = NULL;
6715         uint32_t rw_act_num = 0;
6716         uint64_t is_root;
6717         const struct mlx5_flow_tunnel *tunnel;
6718         enum mlx5_tof_rule_type tof_rule_type;
6719         struct flow_grp_info grp_info = {
6720                 .external = !!external,
6721                 .transfer = !!attr->transfer,
6722                 .fdb_def_rule = !!priv->fdb_def_rule,
6723                 .std_tbl_fix = true,
6724         };
6725         const struct rte_eth_hairpin_conf *conf;
6726         const struct rte_flow_item *rule_items = items;
6727         const struct rte_flow_item *port_id_item = NULL;
6728         bool def_policy = false;
6729
6730         if (items == NULL)
6731                 return -1;
6732         tunnel = is_tunnel_offload_active(dev) ?
6733                  mlx5_get_tof(items, actions, &tof_rule_type) : NULL;
6734         if (tunnel) {
6735                 if (priv->representor)
6736                         return rte_flow_error_set
6737                                 (error, ENOTSUP,
6738                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6739                                  NULL, "decap not supported for VF representor");
6740                 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE)
6741                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6742                 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE)
6743                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
6744                                         MLX5_FLOW_ACTION_DECAP;
6745                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
6746                                         (dev, attr, tunnel, tof_rule_type);
6747         }
6748         ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error);
6749         if (ret < 0)
6750                 return ret;
6751         is_root = (uint64_t)ret;
6752         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6753                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6754                 int type = items->type;
6755
6756                 if (!mlx5_flow_os_item_supported(type))
6757                         return rte_flow_error_set(error, ENOTSUP,
6758                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6759                                                   NULL, "item not supported");
6760                 switch (type) {
6761                 case RTE_FLOW_ITEM_TYPE_VOID:
6762                         break;
6763                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6764                         ret = flow_dv_validate_item_port_id
6765                                         (dev, items, attr, item_flags, error);
6766                         if (ret < 0)
6767                                 return ret;
6768                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6769                         port_id_item = items;
6770                         break;
6771                 case RTE_FLOW_ITEM_TYPE_ETH:
6772                         ret = mlx5_flow_validate_item_eth(items, item_flags,
6773                                                           true, error);
6774                         if (ret < 0)
6775                                 return ret;
6776                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
6777                                              MLX5_FLOW_LAYER_OUTER_L2;
6778                         if (items->mask != NULL && items->spec != NULL) {
6779                                 ether_type =
6780                                         ((const struct rte_flow_item_eth *)
6781                                          items->spec)->type;
6782                                 ether_type &=
6783                                         ((const struct rte_flow_item_eth *)
6784                                          items->mask)->type;
6785                                 ether_type = rte_be_to_cpu_16(ether_type);
6786                         } else {
6787                                 ether_type = 0;
6788                         }
6789                         break;
6790                 case RTE_FLOW_ITEM_TYPE_VLAN:
6791                         ret = flow_dv_validate_item_vlan(items, item_flags,
6792                                                          dev, error);
6793                         if (ret < 0)
6794                                 return ret;
6795                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
6796                                              MLX5_FLOW_LAYER_OUTER_VLAN;
6797                         if (items->mask != NULL && items->spec != NULL) {
6798                                 ether_type =
6799                                         ((const struct rte_flow_item_vlan *)
6800                                          items->spec)->inner_type;
6801                                 ether_type &=
6802                                         ((const struct rte_flow_item_vlan *)
6803                                          items->mask)->inner_type;
6804                                 ether_type = rte_be_to_cpu_16(ether_type);
6805                         } else {
6806                                 ether_type = 0;
6807                         }
6808                         /* Store outer VLAN mask for of_push_vlan action. */
6809                         if (!tunnel)
6810                                 vlan_m = items->mask;
6811                         break;
6812                 case RTE_FLOW_ITEM_TYPE_IPV4:
6813                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6814                                                   &item_flags, &tunnel);
6815                         ret = flow_dv_validate_item_ipv4(items, item_flags,
6816                                                          last_item, ether_type,
6817                                                          error);
6818                         if (ret < 0)
6819                                 return ret;
6820                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
6821                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
6822                         if (items->mask != NULL &&
6823                             ((const struct rte_flow_item_ipv4 *)
6824                              items->mask)->hdr.next_proto_id) {
6825                                 next_protocol =
6826                                         ((const struct rte_flow_item_ipv4 *)
6827                                          (items->spec))->hdr.next_proto_id;
6828                                 next_protocol &=
6829                                         ((const struct rte_flow_item_ipv4 *)
6830                                          (items->mask))->hdr.next_proto_id;
6831                         } else {
6832                                 /* Reset for inner layer. */
6833                                 next_protocol = 0xff;
6834                         }
6835                         break;
6836                 case RTE_FLOW_ITEM_TYPE_IPV6:
6837                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6838                                                   &item_flags, &tunnel);
6839                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
6840                                                            last_item,
6841                                                            ether_type,
6842                                                            &nic_ipv6_mask,
6843                                                            error);
6844                         if (ret < 0)
6845                                 return ret;
6846                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
6847                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
6848                         if (items->mask != NULL &&
6849                             ((const struct rte_flow_item_ipv6 *)
6850                              items->mask)->hdr.proto) {
6851                                 item_ipv6_proto =
6852                                         ((const struct rte_flow_item_ipv6 *)
6853                                          items->spec)->hdr.proto;
6854                                 next_protocol =
6855                                         ((const struct rte_flow_item_ipv6 *)
6856                                          items->spec)->hdr.proto;
6857                                 next_protocol &=
6858                                         ((const struct rte_flow_item_ipv6 *)
6859                                          items->mask)->hdr.proto;
6860                         } else {
6861                                 /* Reset for inner layer. */
6862                                 next_protocol = 0xff;
6863                         }
6864                         break;
6865                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
6866                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
6867                                                                   item_flags,
6868                                                                   error);
6869                         if (ret < 0)
6870                                 return ret;
6871                         last_item = tunnel ?
6872                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
6873                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
6874                         if (items->mask != NULL &&
6875                             ((const struct rte_flow_item_ipv6_frag_ext *)
6876                              items->mask)->hdr.next_header) {
6877                                 next_protocol =
6878                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6879                                  items->spec)->hdr.next_header;
6880                                 next_protocol &=
6881                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6882                                  items->mask)->hdr.next_header;
6883                         } else {
6884                                 /* Reset for inner layer. */
6885                                 next_protocol = 0xff;
6886                         }
6887                         break;
6888                 case RTE_FLOW_ITEM_TYPE_TCP:
6889                         ret = mlx5_flow_validate_item_tcp
6890                                                 (items, item_flags,
6891                                                  next_protocol,
6892                                                  &nic_tcp_mask,
6893                                                  error);
6894                         if (ret < 0)
6895                                 return ret;
6896                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
6897                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
6898                         break;
6899                 case RTE_FLOW_ITEM_TYPE_UDP:
6900                         ret = mlx5_flow_validate_item_udp(items, item_flags,
6901                                                           next_protocol,
6902                                                           error);
6903                         if (ret < 0)
6904                                 return ret;
6905                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
6906                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
6907                         break;
6908                 case RTE_FLOW_ITEM_TYPE_GRE:
6909                         ret = mlx5_flow_validate_item_gre(items, item_flags,
6910                                                           next_protocol, error);
6911                         if (ret < 0)
6912                                 return ret;
6913                         gre_item = items;
6914                         last_item = MLX5_FLOW_LAYER_GRE;
6915                         break;
6916                 case RTE_FLOW_ITEM_TYPE_NVGRE:
6917                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
6918                                                             next_protocol,
6919                                                             error);
6920                         if (ret < 0)
6921                                 return ret;
6922                         last_item = MLX5_FLOW_LAYER_NVGRE;
6923                         break;
6924                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
6925                         ret = mlx5_flow_validate_item_gre_key
6926                                 (items, item_flags, gre_item, error);
6927                         if (ret < 0)
6928                                 return ret;
6929                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
6930                         break;
6931                 case RTE_FLOW_ITEM_TYPE_VXLAN:
6932                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
6933                                                             error);
6934                         if (ret < 0)
6935                                 return ret;
6936                         last_item = MLX5_FLOW_LAYER_VXLAN;
6937                         break;
6938                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
6939                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
6940                                                                 item_flags, dev,
6941                                                                 error);
6942                         if (ret < 0)
6943                                 return ret;
6944                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
6945                         break;
6946                 case RTE_FLOW_ITEM_TYPE_GENEVE:
6947                         ret = mlx5_flow_validate_item_geneve(items,
6948                                                              item_flags, dev,
6949                                                              error);
6950                         if (ret < 0)
6951                                 return ret;
6952                         geneve_item = items;
6953                         last_item = MLX5_FLOW_LAYER_GENEVE;
6954                         break;
6955                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
6956                         ret = mlx5_flow_validate_item_geneve_opt(items,
6957                                                                  last_item,
6958                                                                  geneve_item,
6959                                                                  dev,
6960                                                                  error);
6961                         if (ret < 0)
6962                                 return ret;
6963                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
6964                         break;
6965                 case RTE_FLOW_ITEM_TYPE_MPLS:
6966                         ret = mlx5_flow_validate_item_mpls(dev, items,
6967                                                            item_flags,
6968                                                            last_item, error);
6969                         if (ret < 0)
6970                                 return ret;
6971                         last_item = MLX5_FLOW_LAYER_MPLS;
6972                         break;
6973
6974                 case RTE_FLOW_ITEM_TYPE_MARK:
6975                         ret = flow_dv_validate_item_mark(dev, items, attr,
6976                                                          error);
6977                         if (ret < 0)
6978                                 return ret;
6979                         last_item = MLX5_FLOW_ITEM_MARK;
6980                         break;
6981                 case RTE_FLOW_ITEM_TYPE_META:
6982                         ret = flow_dv_validate_item_meta(dev, items, attr,
6983                                                          error);
6984                         if (ret < 0)
6985                                 return ret;
6986                         last_item = MLX5_FLOW_ITEM_METADATA;
6987                         break;
6988                 case RTE_FLOW_ITEM_TYPE_ICMP:
6989                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
6990                                                            next_protocol,
6991                                                            error);
6992                         if (ret < 0)
6993                                 return ret;
6994                         last_item = MLX5_FLOW_LAYER_ICMP;
6995                         break;
6996                 case RTE_FLOW_ITEM_TYPE_ICMP6:
6997                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
6998                                                             next_protocol,
6999                                                             error);
7000                         if (ret < 0)
7001                                 return ret;
7002                         item_ipv6_proto = IPPROTO_ICMPV6;
7003                         last_item = MLX5_FLOW_LAYER_ICMP6;
7004                         break;
7005                 case RTE_FLOW_ITEM_TYPE_TAG:
7006                         ret = flow_dv_validate_item_tag(dev, items,
7007                                                         attr, error);
7008                         if (ret < 0)
7009                                 return ret;
7010                         last_item = MLX5_FLOW_ITEM_TAG;
7011                         break;
7012                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
7013                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
7014                         break;
7015                 case RTE_FLOW_ITEM_TYPE_GTP:
7016                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
7017                                                         error);
7018                         if (ret < 0)
7019                                 return ret;
7020                         gtp_item = items;
7021                         last_item = MLX5_FLOW_LAYER_GTP;
7022                         break;
7023                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
7024                         ret = flow_dv_validate_item_gtp_psc(items, last_item,
7025                                                             gtp_item, attr,
7026                                                             error);
7027                         if (ret < 0)
7028                                 return ret;
7029                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
7030                         break;
7031                 case RTE_FLOW_ITEM_TYPE_ECPRI:
7032                         /* Capacity will be checked in the translate stage. */
7033                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
7034                                                             last_item,
7035                                                             ether_type,
7036                                                             &nic_ecpri_mask,
7037                                                             error);
7038                         if (ret < 0)
7039                                 return ret;
7040                         last_item = MLX5_FLOW_LAYER_ECPRI;
7041                         break;
7042                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
7043                         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY)
7044                                 return rte_flow_error_set
7045                                         (error, ENOTSUP,
7046                                          RTE_FLOW_ERROR_TYPE_ITEM,
7047                                          NULL, "multiple integrity items not supported");
7048                         ret = flow_dv_validate_item_integrity(dev, rule_items,
7049                                                               items, error);
7050                         if (ret < 0)
7051                                 return ret;
7052                         last_item = MLX5_FLOW_ITEM_INTEGRITY;
7053                         break;
7054                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
7055                         ret = flow_dv_validate_item_aso_ct(dev, items,
7056                                                            &item_flags, error);
7057                         if (ret < 0)
7058                                 return ret;
7059                         break;
7060                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
7061                         /* tunnel offload item was processed before
7062                          * list it here as a supported type
7063                          */
7064                         break;
7065                 default:
7066                         return rte_flow_error_set(error, ENOTSUP,
7067                                                   RTE_FLOW_ERROR_TYPE_ITEM,
7068                                                   NULL, "item not supported");
7069                 }
7070                 item_flags |= last_item;
7071         }
7072         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7073                 int type = actions->type;
7074                 bool shared_count = false;
7075
7076                 if (!mlx5_flow_os_action_supported(type))
7077                         return rte_flow_error_set(error, ENOTSUP,
7078                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7079                                                   actions,
7080                                                   "action not supported");
7081                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
7082                         return rte_flow_error_set(error, ENOTSUP,
7083                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7084                                                   actions, "too many actions");
7085                 if (action_flags &
7086                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
7087                         return rte_flow_error_set(error, ENOTSUP,
7088                                 RTE_FLOW_ERROR_TYPE_ACTION,
7089                                 NULL, "meter action with policy "
7090                                 "must be the last action");
7091                 switch (type) {
7092                 case RTE_FLOW_ACTION_TYPE_VOID:
7093                         break;
7094                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7095                         ret = flow_dv_validate_action_port_id(dev,
7096                                                               action_flags,
7097                                                               actions,
7098                                                               attr,
7099                                                               error);
7100                         if (ret)
7101                                 return ret;
7102                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7103                         ++actions_n;
7104                         break;
7105                 case RTE_FLOW_ACTION_TYPE_FLAG:
7106                         ret = flow_dv_validate_action_flag(dev, action_flags,
7107                                                            attr, error);
7108                         if (ret < 0)
7109                                 return ret;
7110                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7111                                 /* Count all modify-header actions as one. */
7112                                 if (!(action_flags &
7113                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7114                                         ++actions_n;
7115                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
7116                                                 MLX5_FLOW_ACTION_MARK_EXT;
7117                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7118                                         modify_after_mirror = 1;
7119
7120                         } else {
7121                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
7122                                 ++actions_n;
7123                         }
7124                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7125                         break;
7126                 case RTE_FLOW_ACTION_TYPE_MARK:
7127                         ret = flow_dv_validate_action_mark(dev, actions,
7128                                                            action_flags,
7129                                                            attr, error);
7130                         if (ret < 0)
7131                                 return ret;
7132                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7133                                 /* Count all modify-header actions as one. */
7134                                 if (!(action_flags &
7135                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7136                                         ++actions_n;
7137                                 action_flags |= MLX5_FLOW_ACTION_MARK |
7138                                                 MLX5_FLOW_ACTION_MARK_EXT;
7139                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7140                                         modify_after_mirror = 1;
7141                         } else {
7142                                 action_flags |= MLX5_FLOW_ACTION_MARK;
7143                                 ++actions_n;
7144                         }
7145                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7146                         break;
7147                 case RTE_FLOW_ACTION_TYPE_SET_META:
7148                         ret = flow_dv_validate_action_set_meta(dev, actions,
7149                                                                action_flags,
7150                                                                attr, error);
7151                         if (ret < 0)
7152                                 return ret;
7153                         /* Count all modify-header actions as one action. */
7154                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7155                                 ++actions_n;
7156                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7157                                 modify_after_mirror = 1;
7158                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7159                         rw_act_num += MLX5_ACT_NUM_SET_META;
7160                         break;
7161                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7162                         ret = flow_dv_validate_action_set_tag(dev, actions,
7163                                                               action_flags,
7164                                                               attr, error);
7165                         if (ret < 0)
7166                                 return ret;
7167                         /* Count all modify-header actions as one action. */
7168                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7169                                 ++actions_n;
7170                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7171                                 modify_after_mirror = 1;
7172                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7173                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7174                         break;
7175                 case RTE_FLOW_ACTION_TYPE_DROP:
7176                         ret = mlx5_flow_validate_action_drop(action_flags,
7177                                                              attr, error);
7178                         if (ret < 0)
7179                                 return ret;
7180                         action_flags |= MLX5_FLOW_ACTION_DROP;
7181                         ++actions_n;
7182                         break;
7183                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7184                         ret = mlx5_flow_validate_action_queue(actions,
7185                                                               action_flags, dev,
7186                                                               attr, error);
7187                         if (ret < 0)
7188                                 return ret;
7189                         queue_index = ((const struct rte_flow_action_queue *)
7190                                                         (actions->conf))->index;
7191                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7192                         ++actions_n;
7193                         break;
7194                 case RTE_FLOW_ACTION_TYPE_RSS:
7195                         rss = actions->conf;
7196                         ret = mlx5_flow_validate_action_rss(actions,
7197                                                             action_flags, dev,
7198                                                             attr, item_flags,
7199                                                             error);
7200                         if (ret < 0)
7201                                 return ret;
7202                         if (rss && sample_rss &&
7203                             (sample_rss->level != rss->level ||
7204                             sample_rss->types != rss->types))
7205                                 return rte_flow_error_set(error, ENOTSUP,
7206                                         RTE_FLOW_ERROR_TYPE_ACTION,
7207                                         NULL,
7208                                         "Can't use the different RSS types "
7209                                         "or level in the same flow");
7210                         if (rss != NULL && rss->queue_num)
7211                                 queue_index = rss->queue[0];
7212                         action_flags |= MLX5_FLOW_ACTION_RSS;
7213                         ++actions_n;
7214                         break;
7215                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
7216                         ret =
7217                         mlx5_flow_validate_action_default_miss(action_flags,
7218                                         attr, error);
7219                         if (ret < 0)
7220                                 return ret;
7221                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
7222                         ++actions_n;
7223                         break;
7224                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
7225                 case RTE_FLOW_ACTION_TYPE_COUNT:
7226                         shared_count = is_shared_action_count(actions);
7227                         ret = flow_dv_validate_action_count(dev, shared_count,
7228                                                             action_flags,
7229                                                             error);
7230                         if (ret < 0)
7231                                 return ret;
7232                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7233                         ++actions_n;
7234                         break;
7235                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7236                         if (flow_dv_validate_action_pop_vlan(dev,
7237                                                              action_flags,
7238                                                              actions,
7239                                                              item_flags, attr,
7240                                                              error))
7241                                 return -rte_errno;
7242                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7243                                 modify_after_mirror = 1;
7244                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7245                         ++actions_n;
7246                         break;
7247                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
7248                         ret = flow_dv_validate_action_push_vlan(dev,
7249                                                                 action_flags,
7250                                                                 vlan_m,
7251                                                                 actions, attr,
7252                                                                 error);
7253                         if (ret < 0)
7254                                 return ret;
7255                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7256                                 modify_after_mirror = 1;
7257                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
7258                         ++actions_n;
7259                         break;
7260                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
7261                         ret = flow_dv_validate_action_set_vlan_pcp
7262                                                 (action_flags, actions, error);
7263                         if (ret < 0)
7264                                 return ret;
7265                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7266                                 modify_after_mirror = 1;
7267                         /* Count PCP with push_vlan command. */
7268                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
7269                         break;
7270                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
7271                         ret = flow_dv_validate_action_set_vlan_vid
7272                                                 (item_flags, action_flags,
7273                                                  actions, error);
7274                         if (ret < 0)
7275                                 return ret;
7276                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7277                                 modify_after_mirror = 1;
7278                         /* Count VID with push_vlan command. */
7279                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
7280                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
7281                         break;
7282                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
7283                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
7284                         ret = flow_dv_validate_action_l2_encap(dev,
7285                                                                action_flags,
7286                                                                actions, attr,
7287                                                                error);
7288                         if (ret < 0)
7289                                 return ret;
7290                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
7291                         ++actions_n;
7292                         break;
7293                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
7294                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
7295                         ret = flow_dv_validate_action_decap(dev, action_flags,
7296                                                             actions, item_flags,
7297                                                             attr, error);
7298                         if (ret < 0)
7299                                 return ret;
7300                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7301                                 modify_after_mirror = 1;
7302                         action_flags |= MLX5_FLOW_ACTION_DECAP;
7303                         ++actions_n;
7304                         break;
7305                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
7306                         ret = flow_dv_validate_action_raw_encap_decap
7307                                 (dev, NULL, actions->conf, attr, &action_flags,
7308                                  &actions_n, actions, item_flags, error);
7309                         if (ret < 0)
7310                                 return ret;
7311                         break;
7312                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
7313                         decap = actions->conf;
7314                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
7315                                 ;
7316                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
7317                                 encap = NULL;
7318                                 actions--;
7319                         } else {
7320                                 encap = actions->conf;
7321                         }
7322                         ret = flow_dv_validate_action_raw_encap_decap
7323                                            (dev,
7324                                             decap ? decap : &empty_decap, encap,
7325                                             attr, &action_flags, &actions_n,
7326                                             actions, item_flags, error);
7327                         if (ret < 0)
7328                                 return ret;
7329                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7330                             (action_flags & MLX5_FLOW_ACTION_DECAP))
7331                                 modify_after_mirror = 1;
7332                         break;
7333                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
7334                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
7335                         ret = flow_dv_validate_action_modify_mac(action_flags,
7336                                                                  actions,
7337                                                                  item_flags,
7338                                                                  error);
7339                         if (ret < 0)
7340                                 return ret;
7341                         /* Count all modify-header actions as one action. */
7342                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7343                                 ++actions_n;
7344                         action_flags |= actions->type ==
7345                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
7346                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
7347                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
7348                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7349                                 modify_after_mirror = 1;
7350                         /*
7351                          * Even if the source and destination MAC addresses have
7352                          * overlap in the header with 4B alignment, the convert
7353                          * function will handle them separately and 4 SW actions
7354                          * will be created. And 2 actions will be added each
7355                          * time no matter how many bytes of address will be set.
7356                          */
7357                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
7358                         break;
7359                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
7360                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
7361                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
7362                                                                   actions,
7363                                                                   item_flags,
7364                                                                   error);
7365                         if (ret < 0)
7366                                 return ret;
7367                         /* Count all modify-header actions as one action. */
7368                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7369                                 ++actions_n;
7370                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7371                                 modify_after_mirror = 1;
7372                         action_flags |= actions->type ==
7373                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
7374                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
7375                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
7376                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
7377                         break;
7378                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
7379                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
7380                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
7381                                                                   actions,
7382                                                                   item_flags,
7383                                                                   error);
7384                         if (ret < 0)
7385                                 return ret;
7386                         if (item_ipv6_proto == IPPROTO_ICMPV6)
7387                                 return rte_flow_error_set(error, ENOTSUP,
7388                                         RTE_FLOW_ERROR_TYPE_ACTION,
7389                                         actions,
7390                                         "Can't change header "
7391                                         "with ICMPv6 proto");
7392                         /* Count all modify-header actions as one action. */
7393                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7394                                 ++actions_n;
7395                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7396                                 modify_after_mirror = 1;
7397                         action_flags |= actions->type ==
7398                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
7399                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
7400                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
7401                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
7402                         break;
7403                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
7404                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
7405                         ret = flow_dv_validate_action_modify_tp(action_flags,
7406                                                                 actions,
7407                                                                 item_flags,
7408                                                                 error);
7409                         if (ret < 0)
7410                                 return ret;
7411                         /* Count all modify-header actions as one action. */
7412                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7413                                 ++actions_n;
7414                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7415                                 modify_after_mirror = 1;
7416                         action_flags |= actions->type ==
7417                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
7418                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
7419                                                 MLX5_FLOW_ACTION_SET_TP_DST;
7420                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
7421                         break;
7422                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
7423                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
7424                         ret = flow_dv_validate_action_modify_ttl(action_flags,
7425                                                                  actions,
7426                                                                  item_flags,
7427                                                                  error);
7428                         if (ret < 0)
7429                                 return ret;
7430                         /* Count all modify-header actions as one action. */
7431                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7432                                 ++actions_n;
7433                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7434                                 modify_after_mirror = 1;
7435                         action_flags |= actions->type ==
7436                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
7437                                                 MLX5_FLOW_ACTION_SET_TTL :
7438                                                 MLX5_FLOW_ACTION_DEC_TTL;
7439                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
7440                         break;
7441                 case RTE_FLOW_ACTION_TYPE_JUMP:
7442                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
7443                                                            action_flags,
7444                                                            attr, external,
7445                                                            error);
7446                         if (ret)
7447                                 return ret;
7448                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7449                             fdb_mirror_limit)
7450                                 return rte_flow_error_set(error, EINVAL,
7451                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7452                                                   NULL,
7453                                                   "sample and jump action combination is not supported");
7454                         ++actions_n;
7455                         action_flags |= MLX5_FLOW_ACTION_JUMP;
7456                         break;
7457                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
7458                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
7459                         ret = flow_dv_validate_action_modify_tcp_seq
7460                                                                 (action_flags,
7461                                                                  actions,
7462                                                                  item_flags,
7463                                                                  error);
7464                         if (ret < 0)
7465                                 return ret;
7466                         /* Count all modify-header actions as one action. */
7467                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7468                                 ++actions_n;
7469                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7470                                 modify_after_mirror = 1;
7471                         action_flags |= actions->type ==
7472                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
7473                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
7474                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
7475                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
7476                         break;
7477                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
7478                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
7479                         ret = flow_dv_validate_action_modify_tcp_ack
7480                                                                 (action_flags,
7481                                                                  actions,
7482                                                                  item_flags,
7483                                                                  error);
7484                         if (ret < 0)
7485                                 return ret;
7486                         /* Count all modify-header actions as one action. */
7487                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7488                                 ++actions_n;
7489                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7490                                 modify_after_mirror = 1;
7491                         action_flags |= actions->type ==
7492                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
7493                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
7494                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
7495                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
7496                         break;
7497                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7498                         break;
7499                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
7500                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7501                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7502                         break;
7503                 case RTE_FLOW_ACTION_TYPE_METER:
7504                         ret = mlx5_flow_validate_action_meter(dev,
7505                                                               action_flags,
7506                                                               actions, attr,
7507                                                               port_id_item,
7508                                                               &def_policy,
7509                                                               error);
7510                         if (ret < 0)
7511                                 return ret;
7512                         action_flags |= MLX5_FLOW_ACTION_METER;
7513                         if (!def_policy)
7514                                 action_flags |=
7515                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
7516                         ++actions_n;
7517                         /* Meter action will add one more TAG action. */
7518                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7519                         break;
7520                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
7521                         if (!attr->transfer && !attr->group)
7522                                 return rte_flow_error_set(error, ENOTSUP,
7523                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7524                                                                            NULL,
7525                           "Shared ASO age action is not supported for group 0");
7526                         if (action_flags & MLX5_FLOW_ACTION_AGE)
7527                                 return rte_flow_error_set
7528                                                   (error, EINVAL,
7529                                                    RTE_FLOW_ERROR_TYPE_ACTION,
7530                                                    NULL,
7531                                                    "duplicate age actions set");
7532                         action_flags |= MLX5_FLOW_ACTION_AGE;
7533                         ++actions_n;
7534                         break;
7535                 case RTE_FLOW_ACTION_TYPE_AGE:
7536                         ret = flow_dv_validate_action_age(action_flags,
7537                                                           actions, dev,
7538                                                           error);
7539                         if (ret < 0)
7540                                 return ret;
7541                         /*
7542                          * Validate the regular AGE action (using counter)
7543                          * mutual exclusion with share counter actions.
7544                          */
7545                         if (!priv->sh->flow_hit_aso_en) {
7546                                 if (shared_count)
7547                                         return rte_flow_error_set
7548                                                 (error, EINVAL,
7549                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7550                                                 NULL,
7551                                                 "old age and shared count combination is not supported");
7552                                 if (sample_count)
7553                                         return rte_flow_error_set
7554                                                 (error, EINVAL,
7555                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7556                                                 NULL,
7557                                                 "old age action and count must be in the same sub flow");
7558                         }
7559                         action_flags |= MLX5_FLOW_ACTION_AGE;
7560                         ++actions_n;
7561                         break;
7562                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
7563                         ret = flow_dv_validate_action_modify_ipv4_dscp
7564                                                          (action_flags,
7565                                                           actions,
7566                                                           item_flags,
7567                                                           error);
7568                         if (ret < 0)
7569                                 return ret;
7570                         /* Count all modify-header actions as one action. */
7571                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7572                                 ++actions_n;
7573                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7574                                 modify_after_mirror = 1;
7575                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
7576                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7577                         break;
7578                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
7579                         ret = flow_dv_validate_action_modify_ipv6_dscp
7580                                                                 (action_flags,
7581                                                                  actions,
7582                                                                  item_flags,
7583                                                                  error);
7584                         if (ret < 0)
7585                                 return ret;
7586                         /* Count all modify-header actions as one action. */
7587                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7588                                 ++actions_n;
7589                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7590                                 modify_after_mirror = 1;
7591                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
7592                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7593                         break;
7594                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
7595                         ret = flow_dv_validate_action_sample(&action_flags,
7596                                                              actions, dev,
7597                                                              attr, item_flags,
7598                                                              rss, &sample_rss,
7599                                                              &sample_count,
7600                                                              &fdb_mirror_limit,
7601                                                              error);
7602                         if (ret < 0)
7603                                 return ret;
7604                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
7605                         ++actions_n;
7606                         break;
7607                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
7608                         ret = flow_dv_validate_action_modify_field(dev,
7609                                                                    action_flags,
7610                                                                    actions,
7611                                                                    attr,
7612                                                                    error);
7613                         if (ret < 0)
7614                                 return ret;
7615                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7616                                 modify_after_mirror = 1;
7617                         /* Count all modify-header actions as one action. */
7618                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7619                                 ++actions_n;
7620                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
7621                         rw_act_num += ret;
7622                         break;
7623                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
7624                         ret = flow_dv_validate_action_aso_ct(dev, action_flags,
7625                                                              item_flags, attr,
7626                                                              error);
7627                         if (ret < 0)
7628                                 return ret;
7629                         action_flags |= MLX5_FLOW_ACTION_CT;
7630                         break;
7631                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
7632                         /* tunnel offload action was processed before
7633                          * list it here as a supported type
7634                          */
7635                         break;
7636                 default:
7637                         return rte_flow_error_set(error, ENOTSUP,
7638                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7639                                                   actions,
7640                                                   "action not supported");
7641                 }
7642         }
7643         /*
7644          * Validate actions in flow rules
7645          * - Explicit decap action is prohibited by the tunnel offload API.
7646          * - Drop action in tunnel steer rule is prohibited by the API.
7647          * - Application cannot use MARK action because it's value can mask
7648          *   tunnel default miss nitification.
7649          * - JUMP in tunnel match rule has no support in current PMD
7650          *   implementation.
7651          * - TAG & META are reserved for future uses.
7652          */
7653         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
7654                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
7655                                             MLX5_FLOW_ACTION_MARK     |
7656                                             MLX5_FLOW_ACTION_SET_TAG  |
7657                                             MLX5_FLOW_ACTION_SET_META |
7658                                             MLX5_FLOW_ACTION_DROP;
7659
7660                 if (action_flags & bad_actions_mask)
7661                         return rte_flow_error_set
7662                                         (error, EINVAL,
7663                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7664                                         "Invalid RTE action in tunnel "
7665                                         "set decap rule");
7666                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
7667                         return rte_flow_error_set
7668                                         (error, EINVAL,
7669                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7670                                         "tunnel set decap rule must terminate "
7671                                         "with JUMP");
7672                 if (!attr->ingress)
7673                         return rte_flow_error_set
7674                                         (error, EINVAL,
7675                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7676                                         "tunnel flows for ingress traffic only");
7677         }
7678         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
7679                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
7680                                             MLX5_FLOW_ACTION_MARK    |
7681                                             MLX5_FLOW_ACTION_SET_TAG |
7682                                             MLX5_FLOW_ACTION_SET_META;
7683
7684                 if (action_flags & bad_actions_mask)
7685                         return rte_flow_error_set
7686                                         (error, EINVAL,
7687                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7688                                         "Invalid RTE action in tunnel "
7689                                         "set match rule");
7690         }
7691         /*
7692          * Validate the drop action mutual exclusion with other actions.
7693          * Drop action is mutually-exclusive with any other action, except for
7694          * Count action.
7695          * Drop action compatibility with tunnel offload was already validated.
7696          */
7697         if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH |
7698                             MLX5_FLOW_ACTION_TUNNEL_MATCH));
7699         else if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
7700             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
7701                 return rte_flow_error_set(error, EINVAL,
7702                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7703                                           "Drop action is mutually-exclusive "
7704                                           "with any other action, except for "
7705                                           "Count action");
7706         /* Eswitch has few restrictions on using items and actions */
7707         if (attr->transfer) {
7708                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7709                     action_flags & MLX5_FLOW_ACTION_FLAG)
7710                         return rte_flow_error_set(error, ENOTSUP,
7711                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7712                                                   NULL,
7713                                                   "unsupported action FLAG");
7714                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7715                     action_flags & MLX5_FLOW_ACTION_MARK)
7716                         return rte_flow_error_set(error, ENOTSUP,
7717                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7718                                                   NULL,
7719                                                   "unsupported action MARK");
7720                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
7721                         return rte_flow_error_set(error, ENOTSUP,
7722                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7723                                                   NULL,
7724                                                   "unsupported action QUEUE");
7725                 if (action_flags & MLX5_FLOW_ACTION_RSS)
7726                         return rte_flow_error_set(error, ENOTSUP,
7727                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7728                                                   NULL,
7729                                                   "unsupported action RSS");
7730                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
7731                         return rte_flow_error_set(error, EINVAL,
7732                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7733                                                   actions,
7734                                                   "no fate action is found");
7735         } else {
7736                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
7737                         return rte_flow_error_set(error, EINVAL,
7738                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7739                                                   actions,
7740                                                   "no fate action is found");
7741         }
7742         /*
7743          * Continue validation for Xcap and VLAN actions.
7744          * If hairpin is working in explicit TX rule mode, there is no actions
7745          * splitting and the validation of hairpin ingress flow should be the
7746          * same as other standard flows.
7747          */
7748         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
7749                              MLX5_FLOW_VLAN_ACTIONS)) &&
7750             (queue_index == 0xFFFF ||
7751              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN ||
7752              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
7753              conf->tx_explicit != 0))) {
7754                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
7755                     MLX5_FLOW_XCAP_ACTIONS)
7756                         return rte_flow_error_set(error, ENOTSUP,
7757                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7758                                                   NULL, "encap and decap "
7759                                                   "combination aren't supported");
7760                 if (!attr->transfer && attr->ingress) {
7761                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7762                                 return rte_flow_error_set
7763                                                 (error, ENOTSUP,
7764                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7765                                                  NULL, "encap is not supported"
7766                                                  " for ingress traffic");
7767                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7768                                 return rte_flow_error_set
7769                                                 (error, ENOTSUP,
7770                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7771                                                  NULL, "push VLAN action not "
7772                                                  "supported for ingress");
7773                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
7774                                         MLX5_FLOW_VLAN_ACTIONS)
7775                                 return rte_flow_error_set
7776                                                 (error, ENOTSUP,
7777                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7778                                                  NULL, "no support for "
7779                                                  "multiple VLAN actions");
7780                 }
7781         }
7782         if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) {
7783                 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS &
7784                         ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) &&
7785                         attr->ingress)
7786                         return rte_flow_error_set
7787                                 (error, ENOTSUP,
7788                                 RTE_FLOW_ERROR_TYPE_ACTION,
7789                                 NULL, "fate action not supported for "
7790                                 "meter with policy");
7791                 if (attr->egress) {
7792                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
7793                                 return rte_flow_error_set
7794                                         (error, ENOTSUP,
7795                                         RTE_FLOW_ERROR_TYPE_ACTION,
7796                                         NULL, "modify header action in egress "
7797                                         "cannot be done before meter action");
7798                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7799                                 return rte_flow_error_set
7800                                         (error, ENOTSUP,
7801                                         RTE_FLOW_ERROR_TYPE_ACTION,
7802                                         NULL, "encap action in egress "
7803                                         "cannot be done before meter action");
7804                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7805                                 return rte_flow_error_set
7806                                         (error, ENOTSUP,
7807                                         RTE_FLOW_ERROR_TYPE_ACTION,
7808                                         NULL, "push vlan action in egress "
7809                                         "cannot be done before meter action");
7810                 }
7811         }
7812         /*
7813          * Hairpin flow will add one more TAG action in TX implicit mode.
7814          * In TX explicit mode, there will be no hairpin flow ID.
7815          */
7816         if (hairpin > 0)
7817                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7818         /* extra metadata enabled: one more TAG action will be add. */
7819         if (dev_conf->dv_flow_en &&
7820             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
7821             mlx5_flow_ext_mreg_supported(dev))
7822                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7823         if (rw_act_num >
7824                         flow_dv_modify_hdr_action_max(dev, is_root)) {
7825                 return rte_flow_error_set(error, ENOTSUP,
7826                                           RTE_FLOW_ERROR_TYPE_ACTION,
7827                                           NULL, "too many header modify"
7828                                           " actions to support");
7829         }
7830         /* Eswitch egress mirror and modify flow has limitation on CX5 */
7831         if (fdb_mirror_limit && modify_after_mirror)
7832                 return rte_flow_error_set(error, EINVAL,
7833                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7834                                 "sample before modify action is not supported");
7835         return 0;
7836 }
7837
7838 /**
7839  * Internal preparation function. Allocates the DV flow size,
7840  * this size is constant.
7841  *
7842  * @param[in] dev
7843  *   Pointer to the rte_eth_dev structure.
7844  * @param[in] attr
7845  *   Pointer to the flow attributes.
7846  * @param[in] items
7847  *   Pointer to the list of items.
7848  * @param[in] actions
7849  *   Pointer to the list of actions.
7850  * @param[out] error
7851  *   Pointer to the error structure.
7852  *
7853  * @return
7854  *   Pointer to mlx5_flow object on success,
7855  *   otherwise NULL and rte_errno is set.
7856  */
7857 static struct mlx5_flow *
7858 flow_dv_prepare(struct rte_eth_dev *dev,
7859                 const struct rte_flow_attr *attr __rte_unused,
7860                 const struct rte_flow_item items[] __rte_unused,
7861                 const struct rte_flow_action actions[] __rte_unused,
7862                 struct rte_flow_error *error)
7863 {
7864         uint32_t handle_idx = 0;
7865         struct mlx5_flow *dev_flow;
7866         struct mlx5_flow_handle *dev_handle;
7867         struct mlx5_priv *priv = dev->data->dev_private;
7868         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
7869
7870         MLX5_ASSERT(wks);
7871         wks->skip_matcher_reg = 0;
7872         wks->policy = NULL;
7873         wks->final_policy = NULL;
7874         /* In case of corrupting the memory. */
7875         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
7876                 rte_flow_error_set(error, ENOSPC,
7877                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7878                                    "not free temporary device flow");
7879                 return NULL;
7880         }
7881         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
7882                                    &handle_idx);
7883         if (!dev_handle) {
7884                 rte_flow_error_set(error, ENOMEM,
7885                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7886                                    "not enough memory to create flow handle");
7887                 return NULL;
7888         }
7889         MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows));
7890         dev_flow = &wks->flows[wks->flow_idx++];
7891         memset(dev_flow, 0, sizeof(*dev_flow));
7892         dev_flow->handle = dev_handle;
7893         dev_flow->handle_idx = handle_idx;
7894         /*
7895          * In some old rdma-core releases, before continuing, a check of the
7896          * length of matching parameter will be done at first. It needs to use
7897          * the length without misc4 param. If the flow has misc4 support, then
7898          * the length needs to be adjusted accordingly. Each param member is
7899          * aligned with a 64B boundary naturally.
7900          */
7901         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
7902                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
7903         dev_flow->ingress = attr->ingress;
7904         dev_flow->dv.transfer = attr->transfer;
7905         return dev_flow;
7906 }
7907
7908 #ifdef RTE_LIBRTE_MLX5_DEBUG
7909 /**
7910  * Sanity check for match mask and value. Similar to check_valid_spec() in
7911  * kernel driver. If unmasked bit is present in value, it returns failure.
7912  *
7913  * @param match_mask
7914  *   pointer to match mask buffer.
7915  * @param match_value
7916  *   pointer to match value buffer.
7917  *
7918  * @return
7919  *   0 if valid, -EINVAL otherwise.
7920  */
7921 static int
7922 flow_dv_check_valid_spec(void *match_mask, void *match_value)
7923 {
7924         uint8_t *m = match_mask;
7925         uint8_t *v = match_value;
7926         unsigned int i;
7927
7928         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
7929                 if (v[i] & ~m[i]) {
7930                         DRV_LOG(ERR,
7931                                 "match_value differs from match_criteria"
7932                                 " %p[%u] != %p[%u]",
7933                                 match_value, i, match_mask, i);
7934                         return -EINVAL;
7935                 }
7936         }
7937         return 0;
7938 }
7939 #endif
7940
7941 /**
7942  * Add match of ip_version.
7943  *
7944  * @param[in] group
7945  *   Flow group.
7946  * @param[in] headers_v
7947  *   Values header pointer.
7948  * @param[in] headers_m
7949  *   Masks header pointer.
7950  * @param[in] ip_version
7951  *   The IP version to set.
7952  */
7953 static inline void
7954 flow_dv_set_match_ip_version(uint32_t group,
7955                              void *headers_v,
7956                              void *headers_m,
7957                              uint8_t ip_version)
7958 {
7959         if (group == 0)
7960                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
7961         else
7962                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
7963                          ip_version);
7964         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
7965         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
7966         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
7967 }
7968
7969 /**
7970  * Add Ethernet item to matcher and to the value.
7971  *
7972  * @param[in, out] matcher
7973  *   Flow matcher.
7974  * @param[in, out] key
7975  *   Flow matcher value.
7976  * @param[in] item
7977  *   Flow pattern to translate.
7978  * @param[in] inner
7979  *   Item is inner pattern.
7980  */
7981 static void
7982 flow_dv_translate_item_eth(void *matcher, void *key,
7983                            const struct rte_flow_item *item, int inner,
7984                            uint32_t group)
7985 {
7986         const struct rte_flow_item_eth *eth_m = item->mask;
7987         const struct rte_flow_item_eth *eth_v = item->spec;
7988         const struct rte_flow_item_eth nic_mask = {
7989                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
7990                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
7991                 .type = RTE_BE16(0xffff),
7992                 .has_vlan = 0,
7993         };
7994         void *hdrs_m;
7995         void *hdrs_v;
7996         char *l24_v;
7997         unsigned int i;
7998
7999         if (!eth_v)
8000                 return;
8001         if (!eth_m)
8002                 eth_m = &nic_mask;
8003         if (inner) {
8004                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8005                                          inner_headers);
8006                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8007         } else {
8008                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8009                                          outer_headers);
8010                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8011         }
8012         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
8013                &eth_m->dst, sizeof(eth_m->dst));
8014         /* The value must be in the range of the mask. */
8015         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
8016         for (i = 0; i < sizeof(eth_m->dst); ++i)
8017                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
8018         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
8019                &eth_m->src, sizeof(eth_m->src));
8020         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
8021         /* The value must be in the range of the mask. */
8022         for (i = 0; i < sizeof(eth_m->dst); ++i)
8023                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
8024         /*
8025          * HW supports match on one Ethertype, the Ethertype following the last
8026          * VLAN tag of the packet (see PRM).
8027          * Set match on ethertype only if ETH header is not followed by VLAN.
8028          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8029          * ethertype, and use ip_version field instead.
8030          * eCPRI over Ether layer will use type value 0xAEFE.
8031          */
8032         if (eth_m->type == 0xFFFF) {
8033                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
8034                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8035                 switch (eth_v->type) {
8036                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8037                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8038                         return;
8039                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
8040                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8041                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8042                         return;
8043                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8044                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8045                         return;
8046                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8047                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8048                         return;
8049                 default:
8050                         break;
8051                 }
8052         }
8053         if (eth_m->has_vlan) {
8054                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8055                 if (eth_v->has_vlan) {
8056                         /*
8057                          * Here, when also has_more_vlan field in VLAN item is
8058                          * not set, only single-tagged packets will be matched.
8059                          */
8060                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8061                         return;
8062                 }
8063         }
8064         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8065                  rte_be_to_cpu_16(eth_m->type));
8066         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
8067         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
8068 }
8069
8070 /**
8071  * Add VLAN item to matcher and to the value.
8072  *
8073  * @param[in, out] dev_flow
8074  *   Flow descriptor.
8075  * @param[in, out] matcher
8076  *   Flow matcher.
8077  * @param[in, out] key
8078  *   Flow matcher value.
8079  * @param[in] item
8080  *   Flow pattern to translate.
8081  * @param[in] inner
8082  *   Item is inner pattern.
8083  */
8084 static void
8085 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
8086                             void *matcher, void *key,
8087                             const struct rte_flow_item *item,
8088                             int inner, uint32_t group)
8089 {
8090         const struct rte_flow_item_vlan *vlan_m = item->mask;
8091         const struct rte_flow_item_vlan *vlan_v = item->spec;
8092         void *hdrs_m;
8093         void *hdrs_v;
8094         uint16_t tci_m;
8095         uint16_t tci_v;
8096
8097         if (inner) {
8098                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8099                                          inner_headers);
8100                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8101         } else {
8102                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8103                                          outer_headers);
8104                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8105                 /*
8106                  * This is workaround, masks are not supported,
8107                  * and pre-validated.
8108                  */
8109                 if (vlan_v)
8110                         dev_flow->handle->vf_vlan.tag =
8111                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
8112         }
8113         /*
8114          * When VLAN item exists in flow, mark packet as tagged,
8115          * even if TCI is not specified.
8116          */
8117         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
8118                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8119                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8120         }
8121         if (!vlan_v)
8122                 return;
8123         if (!vlan_m)
8124                 vlan_m = &rte_flow_item_vlan_mask;
8125         tci_m = rte_be_to_cpu_16(vlan_m->tci);
8126         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
8127         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
8128         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
8129         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
8130         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
8131         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
8132         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
8133         /*
8134          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8135          * ethertype, and use ip_version field instead.
8136          */
8137         if (vlan_m->inner_type == 0xFFFF) {
8138                 switch (vlan_v->inner_type) {
8139                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8140                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8141                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8142                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8143                         return;
8144                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8145                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8146                         return;
8147                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8148                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8149                         return;
8150                 default:
8151                         break;
8152                 }
8153         }
8154         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
8155                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8156                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8157                 /* Only one vlan_tag bit can be set. */
8158                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8159                 return;
8160         }
8161         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8162                  rte_be_to_cpu_16(vlan_m->inner_type));
8163         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
8164                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
8165 }
8166
8167 /**
8168  * Add IPV4 item to matcher and to the value.
8169  *
8170  * @param[in, out] matcher
8171  *   Flow matcher.
8172  * @param[in, out] key
8173  *   Flow matcher value.
8174  * @param[in] item
8175  *   Flow pattern to translate.
8176  * @param[in] inner
8177  *   Item is inner pattern.
8178  * @param[in] group
8179  *   The group to insert the rule.
8180  */
8181 static void
8182 flow_dv_translate_item_ipv4(void *matcher, void *key,
8183                             const struct rte_flow_item *item,
8184                             int inner, uint32_t group)
8185 {
8186         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
8187         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
8188         const struct rte_flow_item_ipv4 nic_mask = {
8189                 .hdr = {
8190                         .src_addr = RTE_BE32(0xffffffff),
8191                         .dst_addr = RTE_BE32(0xffffffff),
8192                         .type_of_service = 0xff,
8193                         .next_proto_id = 0xff,
8194                         .time_to_live = 0xff,
8195                 },
8196         };
8197         void *headers_m;
8198         void *headers_v;
8199         char *l24_m;
8200         char *l24_v;
8201         uint8_t tos;
8202
8203         if (inner) {
8204                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8205                                          inner_headers);
8206                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8207         } else {
8208                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8209                                          outer_headers);
8210                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8211         }
8212         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
8213         if (!ipv4_v)
8214                 return;
8215         if (!ipv4_m)
8216                 ipv4_m = &nic_mask;
8217         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8218                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8219         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8220                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8221         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
8222         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
8223         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8224                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8225         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8226                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8227         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
8228         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
8229         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
8230         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
8231                  ipv4_m->hdr.type_of_service);
8232         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
8233         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
8234                  ipv4_m->hdr.type_of_service >> 2);
8235         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
8236         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8237                  ipv4_m->hdr.next_proto_id);
8238         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8239                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
8240         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8241                  ipv4_m->hdr.time_to_live);
8242         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8243                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
8244         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8245                  !!(ipv4_m->hdr.fragment_offset));
8246         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8247                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
8248 }
8249
8250 /**
8251  * Add IPV6 item to matcher and to the value.
8252  *
8253  * @param[in, out] matcher
8254  *   Flow matcher.
8255  * @param[in, out] key
8256  *   Flow matcher value.
8257  * @param[in] item
8258  *   Flow pattern to translate.
8259  * @param[in] inner
8260  *   Item is inner pattern.
8261  * @param[in] group
8262  *   The group to insert the rule.
8263  */
8264 static void
8265 flow_dv_translate_item_ipv6(void *matcher, void *key,
8266                             const struct rte_flow_item *item,
8267                             int inner, uint32_t group)
8268 {
8269         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
8270         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
8271         const struct rte_flow_item_ipv6 nic_mask = {
8272                 .hdr = {
8273                         .src_addr =
8274                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8275                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8276                         .dst_addr =
8277                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8278                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8279                         .vtc_flow = RTE_BE32(0xffffffff),
8280                         .proto = 0xff,
8281                         .hop_limits = 0xff,
8282                 },
8283         };
8284         void *headers_m;
8285         void *headers_v;
8286         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8287         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8288         char *l24_m;
8289         char *l24_v;
8290         uint32_t vtc_m;
8291         uint32_t vtc_v;
8292         int i;
8293         int size;
8294
8295         if (inner) {
8296                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8297                                          inner_headers);
8298                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8299         } else {
8300                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8301                                          outer_headers);
8302                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8303         }
8304         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
8305         if (!ipv6_v)
8306                 return;
8307         if (!ipv6_m)
8308                 ipv6_m = &nic_mask;
8309         size = sizeof(ipv6_m->hdr.dst_addr);
8310         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8311                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8312         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8313                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8314         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
8315         for (i = 0; i < size; ++i)
8316                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
8317         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8318                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8319         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8320                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8321         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
8322         for (i = 0; i < size; ++i)
8323                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
8324         /* TOS. */
8325         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
8326         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
8327         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
8328         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
8329         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
8330         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
8331         /* Label. */
8332         if (inner) {
8333                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
8334                          vtc_m);
8335                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
8336                          vtc_v);
8337         } else {
8338                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
8339                          vtc_m);
8340                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
8341                          vtc_v);
8342         }
8343         /* Protocol. */
8344         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8345                  ipv6_m->hdr.proto);
8346         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8347                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
8348         /* Hop limit. */
8349         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8350                  ipv6_m->hdr.hop_limits);
8351         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8352                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
8353         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8354                  !!(ipv6_m->has_frag_ext));
8355         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8356                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
8357 }
8358
8359 /**
8360  * Add IPV6 fragment extension item to matcher and to the value.
8361  *
8362  * @param[in, out] matcher
8363  *   Flow matcher.
8364  * @param[in, out] key
8365  *   Flow matcher value.
8366  * @param[in] item
8367  *   Flow pattern to translate.
8368  * @param[in] inner
8369  *   Item is inner pattern.
8370  */
8371 static void
8372 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
8373                                      const struct rte_flow_item *item,
8374                                      int inner)
8375 {
8376         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
8377         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
8378         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
8379                 .hdr = {
8380                         .next_header = 0xff,
8381                         .frag_data = RTE_BE16(0xffff),
8382                 },
8383         };
8384         void *headers_m;
8385         void *headers_v;
8386
8387         if (inner) {
8388                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8389                                          inner_headers);
8390                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8391         } else {
8392                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8393                                          outer_headers);
8394                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8395         }
8396         /* IPv6 fragment extension item exists, so packet is IP fragment. */
8397         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
8398         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
8399         if (!ipv6_frag_ext_v)
8400                 return;
8401         if (!ipv6_frag_ext_m)
8402                 ipv6_frag_ext_m = &nic_mask;
8403         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8404                  ipv6_frag_ext_m->hdr.next_header);
8405         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8406                  ipv6_frag_ext_v->hdr.next_header &
8407                  ipv6_frag_ext_m->hdr.next_header);
8408 }
8409
8410 /**
8411  * Add TCP item to matcher and to the value.
8412  *
8413  * @param[in, out] matcher
8414  *   Flow matcher.
8415  * @param[in, out] key
8416  *   Flow matcher value.
8417  * @param[in] item
8418  *   Flow pattern to translate.
8419  * @param[in] inner
8420  *   Item is inner pattern.
8421  */
8422 static void
8423 flow_dv_translate_item_tcp(void *matcher, void *key,
8424                            const struct rte_flow_item *item,
8425                            int inner)
8426 {
8427         const struct rte_flow_item_tcp *tcp_m = item->mask;
8428         const struct rte_flow_item_tcp *tcp_v = item->spec;
8429         void *headers_m;
8430         void *headers_v;
8431
8432         if (inner) {
8433                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8434                                          inner_headers);
8435                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8436         } else {
8437                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8438                                          outer_headers);
8439                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8440         }
8441         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8442         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
8443         if (!tcp_v)
8444                 return;
8445         if (!tcp_m)
8446                 tcp_m = &rte_flow_item_tcp_mask;
8447         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
8448                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
8449         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
8450                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
8451         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
8452                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
8453         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
8454                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
8455         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
8456                  tcp_m->hdr.tcp_flags);
8457         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
8458                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
8459 }
8460
8461 /**
8462  * Add UDP item to matcher and to the value.
8463  *
8464  * @param[in, out] matcher
8465  *   Flow matcher.
8466  * @param[in, out] key
8467  *   Flow matcher value.
8468  * @param[in] item
8469  *   Flow pattern to translate.
8470  * @param[in] inner
8471  *   Item is inner pattern.
8472  */
8473 static void
8474 flow_dv_translate_item_udp(void *matcher, void *key,
8475                            const struct rte_flow_item *item,
8476                            int inner)
8477 {
8478         const struct rte_flow_item_udp *udp_m = item->mask;
8479         const struct rte_flow_item_udp *udp_v = item->spec;
8480         void *headers_m;
8481         void *headers_v;
8482
8483         if (inner) {
8484                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8485                                          inner_headers);
8486                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8487         } else {
8488                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8489                                          outer_headers);
8490                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8491         }
8492         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8493         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
8494         if (!udp_v)
8495                 return;
8496         if (!udp_m)
8497                 udp_m = &rte_flow_item_udp_mask;
8498         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
8499                  rte_be_to_cpu_16(udp_m->hdr.src_port));
8500         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
8501                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
8502         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
8503                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
8504         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
8505                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
8506 }
8507
8508 /**
8509  * Add GRE optional Key item to matcher and to the value.
8510  *
8511  * @param[in, out] matcher
8512  *   Flow matcher.
8513  * @param[in, out] key
8514  *   Flow matcher value.
8515  * @param[in] item
8516  *   Flow pattern to translate.
8517  * @param[in] inner
8518  *   Item is inner pattern.
8519  */
8520 static void
8521 flow_dv_translate_item_gre_key(void *matcher, void *key,
8522                                    const struct rte_flow_item *item)
8523 {
8524         const rte_be32_t *key_m = item->mask;
8525         const rte_be32_t *key_v = item->spec;
8526         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8527         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8528         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
8529
8530         /* GRE K bit must be on and should already be validated */
8531         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
8532         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
8533         if (!key_v)
8534                 return;
8535         if (!key_m)
8536                 key_m = &gre_key_default_mask;
8537         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
8538                  rte_be_to_cpu_32(*key_m) >> 8);
8539         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
8540                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
8541         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
8542                  rte_be_to_cpu_32(*key_m) & 0xFF);
8543         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
8544                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
8545 }
8546
8547 /**
8548  * Add GRE item to matcher and to the value.
8549  *
8550  * @param[in, out] matcher
8551  *   Flow matcher.
8552  * @param[in, out] key
8553  *   Flow matcher value.
8554  * @param[in] item
8555  *   Flow pattern to translate.
8556  * @param[in] inner
8557  *   Item is inner pattern.
8558  */
8559 static void
8560 flow_dv_translate_item_gre(void *matcher, void *key,
8561                            const struct rte_flow_item *item,
8562                            int inner)
8563 {
8564         const struct rte_flow_item_gre *gre_m = item->mask;
8565         const struct rte_flow_item_gre *gre_v = item->spec;
8566         void *headers_m;
8567         void *headers_v;
8568         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8569         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8570         struct {
8571                 union {
8572                         __extension__
8573                         struct {
8574                                 uint16_t version:3;
8575                                 uint16_t rsvd0:9;
8576                                 uint16_t s_present:1;
8577                                 uint16_t k_present:1;
8578                                 uint16_t rsvd_bit1:1;
8579                                 uint16_t c_present:1;
8580                         };
8581                         uint16_t value;
8582                 };
8583         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
8584
8585         if (inner) {
8586                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8587                                          inner_headers);
8588                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8589         } else {
8590                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8591                                          outer_headers);
8592                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8593         }
8594         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8595         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
8596         if (!gre_v)
8597                 return;
8598         if (!gre_m)
8599                 gre_m = &rte_flow_item_gre_mask;
8600         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
8601                  rte_be_to_cpu_16(gre_m->protocol));
8602         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
8603                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
8604         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
8605         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
8606         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
8607                  gre_crks_rsvd0_ver_m.c_present);
8608         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
8609                  gre_crks_rsvd0_ver_v.c_present &
8610                  gre_crks_rsvd0_ver_m.c_present);
8611         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
8612                  gre_crks_rsvd0_ver_m.k_present);
8613         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
8614                  gre_crks_rsvd0_ver_v.k_present &
8615                  gre_crks_rsvd0_ver_m.k_present);
8616         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
8617                  gre_crks_rsvd0_ver_m.s_present);
8618         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
8619                  gre_crks_rsvd0_ver_v.s_present &
8620                  gre_crks_rsvd0_ver_m.s_present);
8621 }
8622
8623 /**
8624  * Add NVGRE item to matcher and to the value.
8625  *
8626  * @param[in, out] matcher
8627  *   Flow matcher.
8628  * @param[in, out] key
8629  *   Flow matcher value.
8630  * @param[in] item
8631  *   Flow pattern to translate.
8632  * @param[in] inner
8633  *   Item is inner pattern.
8634  */
8635 static void
8636 flow_dv_translate_item_nvgre(void *matcher, void *key,
8637                              const struct rte_flow_item *item,
8638                              int inner)
8639 {
8640         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
8641         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
8642         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8643         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8644         const char *tni_flow_id_m;
8645         const char *tni_flow_id_v;
8646         char *gre_key_m;
8647         char *gre_key_v;
8648         int size;
8649         int i;
8650
8651         /* For NVGRE, GRE header fields must be set with defined values. */
8652         const struct rte_flow_item_gre gre_spec = {
8653                 .c_rsvd0_ver = RTE_BE16(0x2000),
8654                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
8655         };
8656         const struct rte_flow_item_gre gre_mask = {
8657                 .c_rsvd0_ver = RTE_BE16(0xB000),
8658                 .protocol = RTE_BE16(UINT16_MAX),
8659         };
8660         const struct rte_flow_item gre_item = {
8661                 .spec = &gre_spec,
8662                 .mask = &gre_mask,
8663                 .last = NULL,
8664         };
8665         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
8666         if (!nvgre_v)
8667                 return;
8668         if (!nvgre_m)
8669                 nvgre_m = &rte_flow_item_nvgre_mask;
8670         tni_flow_id_m = (const char *)nvgre_m->tni;
8671         tni_flow_id_v = (const char *)nvgre_v->tni;
8672         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
8673         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
8674         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
8675         memcpy(gre_key_m, tni_flow_id_m, size);
8676         for (i = 0; i < size; ++i)
8677                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
8678 }
8679
8680 /**
8681  * Add VXLAN item to matcher and to the value.
8682  *
8683  * @param[in, out] matcher
8684  *   Flow matcher.
8685  * @param[in, out] key
8686  *   Flow matcher value.
8687  * @param[in] item
8688  *   Flow pattern to translate.
8689  * @param[in] inner
8690  *   Item is inner pattern.
8691  */
8692 static void
8693 flow_dv_translate_item_vxlan(void *matcher, void *key,
8694                              const struct rte_flow_item *item,
8695                              int inner)
8696 {
8697         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
8698         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
8699         void *headers_m;
8700         void *headers_v;
8701         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8702         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8703         char *vni_m;
8704         char *vni_v;
8705         uint16_t dport;
8706         int size;
8707         int i;
8708
8709         if (inner) {
8710                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8711                                          inner_headers);
8712                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8713         } else {
8714                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8715                                          outer_headers);
8716                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8717         }
8718         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8719                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8720         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8721                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8722                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8723         }
8724         if (!vxlan_v)
8725                 return;
8726         if (!vxlan_m)
8727                 vxlan_m = &rte_flow_item_vxlan_mask;
8728         size = sizeof(vxlan_m->vni);
8729         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
8730         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
8731         memcpy(vni_m, vxlan_m->vni, size);
8732         for (i = 0; i < size; ++i)
8733                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8734 }
8735
8736 /**
8737  * Add VXLAN-GPE item to matcher and to the value.
8738  *
8739  * @param[in, out] matcher
8740  *   Flow matcher.
8741  * @param[in, out] key
8742  *   Flow matcher value.
8743  * @param[in] item
8744  *   Flow pattern to translate.
8745  * @param[in] inner
8746  *   Item is inner pattern.
8747  */
8748
8749 static void
8750 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
8751                                  const struct rte_flow_item *item, int inner)
8752 {
8753         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
8754         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
8755         void *headers_m;
8756         void *headers_v;
8757         void *misc_m =
8758                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
8759         void *misc_v =
8760                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
8761         char *vni_m;
8762         char *vni_v;
8763         uint16_t dport;
8764         int size;
8765         int i;
8766         uint8_t flags_m = 0xff;
8767         uint8_t flags_v = 0xc;
8768
8769         if (inner) {
8770                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8771                                          inner_headers);
8772                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8773         } else {
8774                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8775                                          outer_headers);
8776                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8777         }
8778         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8779                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8780         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8781                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8782                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8783         }
8784         if (!vxlan_v)
8785                 return;
8786         if (!vxlan_m)
8787                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
8788         size = sizeof(vxlan_m->vni);
8789         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
8790         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
8791         memcpy(vni_m, vxlan_m->vni, size);
8792         for (i = 0; i < size; ++i)
8793                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8794         if (vxlan_m->flags) {
8795                 flags_m = vxlan_m->flags;
8796                 flags_v = vxlan_v->flags;
8797         }
8798         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
8799         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
8800         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
8801                  vxlan_m->protocol);
8802         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
8803                  vxlan_v->protocol);
8804 }
8805
8806 /**
8807  * Add Geneve item to matcher and to the value.
8808  *
8809  * @param[in, out] matcher
8810  *   Flow matcher.
8811  * @param[in, out] key
8812  *   Flow matcher value.
8813  * @param[in] item
8814  *   Flow pattern to translate.
8815  * @param[in] inner
8816  *   Item is inner pattern.
8817  */
8818
8819 static void
8820 flow_dv_translate_item_geneve(void *matcher, void *key,
8821                               const struct rte_flow_item *item, int inner)
8822 {
8823         const struct rte_flow_item_geneve *geneve_m = item->mask;
8824         const struct rte_flow_item_geneve *geneve_v = item->spec;
8825         void *headers_m;
8826         void *headers_v;
8827         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8828         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8829         uint16_t dport;
8830         uint16_t gbhdr_m;
8831         uint16_t gbhdr_v;
8832         char *vni_m;
8833         char *vni_v;
8834         size_t size, i;
8835
8836         if (inner) {
8837                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8838                                          inner_headers);
8839                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8840         } else {
8841                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8842                                          outer_headers);
8843                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8844         }
8845         dport = MLX5_UDP_PORT_GENEVE;
8846         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8847                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8848                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8849         }
8850         if (!geneve_v)
8851                 return;
8852         if (!geneve_m)
8853                 geneve_m = &rte_flow_item_geneve_mask;
8854         size = sizeof(geneve_m->vni);
8855         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
8856         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
8857         memcpy(vni_m, geneve_m->vni, size);
8858         for (i = 0; i < size; ++i)
8859                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
8860         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
8861                  rte_be_to_cpu_16(geneve_m->protocol));
8862         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
8863                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
8864         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
8865         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
8866         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
8867                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
8868         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
8869                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
8870         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
8871                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
8872         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
8873                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
8874                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
8875 }
8876
8877 /**
8878  * Create Geneve TLV option resource.
8879  *
8880  * @param dev[in, out]
8881  *   Pointer to rte_eth_dev structure.
8882  * @param[in, out] tag_be24
8883  *   Tag value in big endian then R-shift 8.
8884  * @parm[in, out] dev_flow
8885  *   Pointer to the dev_flow.
8886  * @param[out] error
8887  *   pointer to error structure.
8888  *
8889  * @return
8890  *   0 on success otherwise -errno and errno is set.
8891  */
8892
8893 int
8894 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
8895                                              const struct rte_flow_item *item,
8896                                              struct rte_flow_error *error)
8897 {
8898         struct mlx5_priv *priv = dev->data->dev_private;
8899         struct mlx5_dev_ctx_shared *sh = priv->sh;
8900         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
8901                         sh->geneve_tlv_option_resource;
8902         struct mlx5_devx_obj *obj;
8903         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
8904         int ret = 0;
8905
8906         if (!geneve_opt_v)
8907                 return -1;
8908         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
8909         if (geneve_opt_resource != NULL) {
8910                 if (geneve_opt_resource->option_class ==
8911                         geneve_opt_v->option_class &&
8912                         geneve_opt_resource->option_type ==
8913                         geneve_opt_v->option_type &&
8914                         geneve_opt_resource->length ==
8915                         geneve_opt_v->option_len) {
8916                         /* We already have GENVE TLV option obj allocated. */
8917                         __atomic_fetch_add(&geneve_opt_resource->refcnt, 1,
8918                                            __ATOMIC_RELAXED);
8919                 } else {
8920                         ret = rte_flow_error_set(error, ENOMEM,
8921                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8922                                 "Only one GENEVE TLV option supported");
8923                         goto exit;
8924                 }
8925         } else {
8926                 /* Create a GENEVE TLV object and resource. */
8927                 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->ctx,
8928                                 geneve_opt_v->option_class,
8929                                 geneve_opt_v->option_type,
8930                                 geneve_opt_v->option_len);
8931                 if (!obj) {
8932                         ret = rte_flow_error_set(error, ENODATA,
8933                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8934                                 "Failed to create GENEVE TLV Devx object");
8935                         goto exit;
8936                 }
8937                 sh->geneve_tlv_option_resource =
8938                                 mlx5_malloc(MLX5_MEM_ZERO,
8939                                                 sizeof(*geneve_opt_resource),
8940                                                 0, SOCKET_ID_ANY);
8941                 if (!sh->geneve_tlv_option_resource) {
8942                         claim_zero(mlx5_devx_cmd_destroy(obj));
8943                         ret = rte_flow_error_set(error, ENOMEM,
8944                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8945                                 "GENEVE TLV object memory allocation failed");
8946                         goto exit;
8947                 }
8948                 geneve_opt_resource = sh->geneve_tlv_option_resource;
8949                 geneve_opt_resource->obj = obj;
8950                 geneve_opt_resource->option_class = geneve_opt_v->option_class;
8951                 geneve_opt_resource->option_type = geneve_opt_v->option_type;
8952                 geneve_opt_resource->length = geneve_opt_v->option_len;
8953                 __atomic_store_n(&geneve_opt_resource->refcnt, 1,
8954                                 __ATOMIC_RELAXED);
8955         }
8956 exit:
8957         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
8958         return ret;
8959 }
8960
8961 /**
8962  * Add Geneve TLV option item to matcher.
8963  *
8964  * @param[in, out] dev
8965  *   Pointer to rte_eth_dev structure.
8966  * @param[in, out] matcher
8967  *   Flow matcher.
8968  * @param[in, out] key
8969  *   Flow matcher value.
8970  * @param[in] item
8971  *   Flow pattern to translate.
8972  * @param[out] error
8973  *   Pointer to error structure.
8974  */
8975 static int
8976 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher,
8977                                   void *key, const struct rte_flow_item *item,
8978                                   struct rte_flow_error *error)
8979 {
8980         const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask;
8981         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
8982         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8983         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8984         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
8985                         misc_parameters_3);
8986         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
8987         rte_be32_t opt_data_key = 0, opt_data_mask = 0;
8988         int ret = 0;
8989
8990         if (!geneve_opt_v)
8991                 return -1;
8992         if (!geneve_opt_m)
8993                 geneve_opt_m = &rte_flow_item_geneve_opt_mask;
8994         ret = flow_dev_geneve_tlv_option_resource_register(dev, item,
8995                                                            error);
8996         if (ret) {
8997                 DRV_LOG(ERR, "Failed to create geneve_tlv_obj");
8998                 return ret;
8999         }
9000         /*
9001          * Set the option length in GENEVE header if not requested.
9002          * The GENEVE TLV option length is expressed by the option length field
9003          * in the GENEVE header.
9004          * If the option length was not requested but the GENEVE TLV option item
9005          * is present we set the option length field implicitly.
9006          */
9007         if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) {
9008                 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9009                          MLX5_GENEVE_OPTLEN_MASK);
9010                 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9011                          geneve_opt_v->option_len + 1);
9012         }
9013         /* Set the data. */
9014         if (geneve_opt_v->data) {
9015                 memcpy(&opt_data_key, geneve_opt_v->data,
9016                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9017                                 sizeof(opt_data_key)));
9018                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9019                                 sizeof(opt_data_key));
9020                 memcpy(&opt_data_mask, geneve_opt_m->data,
9021                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9022                                 sizeof(opt_data_mask)));
9023                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9024                                 sizeof(opt_data_mask));
9025                 MLX5_SET(fte_match_set_misc3, misc3_m,
9026                                 geneve_tlv_option_0_data,
9027                                 rte_be_to_cpu_32(opt_data_mask));
9028                 MLX5_SET(fte_match_set_misc3, misc3_v,
9029                                 geneve_tlv_option_0_data,
9030                         rte_be_to_cpu_32(opt_data_key & opt_data_mask));
9031         }
9032         return ret;
9033 }
9034
9035 /**
9036  * Add MPLS item to matcher and to the value.
9037  *
9038  * @param[in, out] matcher
9039  *   Flow matcher.
9040  * @param[in, out] key
9041  *   Flow matcher value.
9042  * @param[in] item
9043  *   Flow pattern to translate.
9044  * @param[in] prev_layer
9045  *   The protocol layer indicated in previous item.
9046  * @param[in] inner
9047  *   Item is inner pattern.
9048  */
9049 static void
9050 flow_dv_translate_item_mpls(void *matcher, void *key,
9051                             const struct rte_flow_item *item,
9052                             uint64_t prev_layer,
9053                             int inner)
9054 {
9055         const uint32_t *in_mpls_m = item->mask;
9056         const uint32_t *in_mpls_v = item->spec;
9057         uint32_t *out_mpls_m = 0;
9058         uint32_t *out_mpls_v = 0;
9059         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9060         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9061         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
9062                                      misc_parameters_2);
9063         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9064         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9065         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9066
9067         switch (prev_layer) {
9068         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9069                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
9070                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9071                          MLX5_UDP_PORT_MPLS);
9072                 break;
9073         case MLX5_FLOW_LAYER_GRE:
9074                 /* Fall-through. */
9075         case MLX5_FLOW_LAYER_GRE_KEY:
9076                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
9077                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
9078                          RTE_ETHER_TYPE_MPLS);
9079                 break;
9080         default:
9081                 break;
9082         }
9083         if (!in_mpls_v)
9084                 return;
9085         if (!in_mpls_m)
9086                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
9087         switch (prev_layer) {
9088         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9089                 out_mpls_m =
9090                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9091                                                  outer_first_mpls_over_udp);
9092                 out_mpls_v =
9093                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9094                                                  outer_first_mpls_over_udp);
9095                 break;
9096         case MLX5_FLOW_LAYER_GRE:
9097                 out_mpls_m =
9098                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9099                                                  outer_first_mpls_over_gre);
9100                 out_mpls_v =
9101                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9102                                                  outer_first_mpls_over_gre);
9103                 break;
9104         default:
9105                 /* Inner MPLS not over GRE is not supported. */
9106                 if (!inner) {
9107                         out_mpls_m =
9108                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9109                                                          misc2_m,
9110                                                          outer_first_mpls);
9111                         out_mpls_v =
9112                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9113                                                          misc2_v,
9114                                                          outer_first_mpls);
9115                 }
9116                 break;
9117         }
9118         if (out_mpls_m && out_mpls_v) {
9119                 *out_mpls_m = *in_mpls_m;
9120                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
9121         }
9122 }
9123
9124 /**
9125  * Add metadata register item to matcher
9126  *
9127  * @param[in, out] matcher
9128  *   Flow matcher.
9129  * @param[in, out] key
9130  *   Flow matcher value.
9131  * @param[in] reg_type
9132  *   Type of device metadata register
9133  * @param[in] value
9134  *   Register value
9135  * @param[in] mask
9136  *   Register mask
9137  */
9138 static void
9139 flow_dv_match_meta_reg(void *matcher, void *key,
9140                        enum modify_reg reg_type,
9141                        uint32_t data, uint32_t mask)
9142 {
9143         void *misc2_m =
9144                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
9145         void *misc2_v =
9146                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9147         uint32_t temp;
9148
9149         data &= mask;
9150         switch (reg_type) {
9151         case REG_A:
9152                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
9153                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
9154                 break;
9155         case REG_B:
9156                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
9157                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
9158                 break;
9159         case REG_C_0:
9160                 /*
9161                  * The metadata register C0 field might be divided into
9162                  * source vport index and META item value, we should set
9163                  * this field according to specified mask, not as whole one.
9164                  */
9165                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
9166                 temp |= mask;
9167                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
9168                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
9169                 temp &= ~mask;
9170                 temp |= data;
9171                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
9172                 break;
9173         case REG_C_1:
9174                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
9175                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
9176                 break;
9177         case REG_C_2:
9178                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
9179                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
9180                 break;
9181         case REG_C_3:
9182                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
9183                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
9184                 break;
9185         case REG_C_4:
9186                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
9187                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
9188                 break;
9189         case REG_C_5:
9190                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
9191                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
9192                 break;
9193         case REG_C_6:
9194                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
9195                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
9196                 break;
9197         case REG_C_7:
9198                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
9199                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
9200                 break;
9201         default:
9202                 MLX5_ASSERT(false);
9203                 break;
9204         }
9205 }
9206
9207 /**
9208  * Add MARK item to matcher
9209  *
9210  * @param[in] dev
9211  *   The device to configure through.
9212  * @param[in, out] matcher
9213  *   Flow matcher.
9214  * @param[in, out] key
9215  *   Flow matcher value.
9216  * @param[in] item
9217  *   Flow pattern to translate.
9218  */
9219 static void
9220 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
9221                             void *matcher, void *key,
9222                             const struct rte_flow_item *item)
9223 {
9224         struct mlx5_priv *priv = dev->data->dev_private;
9225         const struct rte_flow_item_mark *mark;
9226         uint32_t value;
9227         uint32_t mask;
9228
9229         mark = item->mask ? (const void *)item->mask :
9230                             &rte_flow_item_mark_mask;
9231         mask = mark->id & priv->sh->dv_mark_mask;
9232         mark = (const void *)item->spec;
9233         MLX5_ASSERT(mark);
9234         value = mark->id & priv->sh->dv_mark_mask & mask;
9235         if (mask) {
9236                 enum modify_reg reg;
9237
9238                 /* Get the metadata register index for the mark. */
9239                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
9240                 MLX5_ASSERT(reg > 0);
9241                 if (reg == REG_C_0) {
9242                         struct mlx5_priv *priv = dev->data->dev_private;
9243                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9244                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9245
9246                         mask &= msk_c0;
9247                         mask <<= shl_c0;
9248                         value <<= shl_c0;
9249                 }
9250                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9251         }
9252 }
9253
9254 /**
9255  * Add META item to matcher
9256  *
9257  * @param[in] dev
9258  *   The devich to configure through.
9259  * @param[in, out] matcher
9260  *   Flow matcher.
9261  * @param[in, out] key
9262  *   Flow matcher value.
9263  * @param[in] attr
9264  *   Attributes of flow that includes this item.
9265  * @param[in] item
9266  *   Flow pattern to translate.
9267  */
9268 static void
9269 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
9270                             void *matcher, void *key,
9271                             const struct rte_flow_attr *attr,
9272                             const struct rte_flow_item *item)
9273 {
9274         const struct rte_flow_item_meta *meta_m;
9275         const struct rte_flow_item_meta *meta_v;
9276
9277         meta_m = (const void *)item->mask;
9278         if (!meta_m)
9279                 meta_m = &rte_flow_item_meta_mask;
9280         meta_v = (const void *)item->spec;
9281         if (meta_v) {
9282                 int reg;
9283                 uint32_t value = meta_v->data;
9284                 uint32_t mask = meta_m->data;
9285
9286                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
9287                 if (reg < 0)
9288                         return;
9289                 MLX5_ASSERT(reg != REG_NON);
9290                 if (reg == REG_C_0) {
9291                         struct mlx5_priv *priv = dev->data->dev_private;
9292                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9293                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9294
9295                         mask &= msk_c0;
9296                         mask <<= shl_c0;
9297                         value <<= shl_c0;
9298                 }
9299                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9300         }
9301 }
9302
9303 /**
9304  * Add vport metadata Reg C0 item to matcher
9305  *
9306  * @param[in, out] matcher
9307  *   Flow matcher.
9308  * @param[in, out] key
9309  *   Flow matcher value.
9310  * @param[in] reg
9311  *   Flow pattern to translate.
9312  */
9313 static void
9314 flow_dv_translate_item_meta_vport(void *matcher, void *key,
9315                                   uint32_t value, uint32_t mask)
9316 {
9317         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
9318 }
9319
9320 /**
9321  * Add tag item to matcher
9322  *
9323  * @param[in] dev
9324  *   The devich to configure through.
9325  * @param[in, out] matcher
9326  *   Flow matcher.
9327  * @param[in, out] key
9328  *   Flow matcher value.
9329  * @param[in] item
9330  *   Flow pattern to translate.
9331  */
9332 static void
9333 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
9334                                 void *matcher, void *key,
9335                                 const struct rte_flow_item *item)
9336 {
9337         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
9338         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
9339         uint32_t mask, value;
9340
9341         MLX5_ASSERT(tag_v);
9342         value = tag_v->data;
9343         mask = tag_m ? tag_m->data : UINT32_MAX;
9344         if (tag_v->id == REG_C_0) {
9345                 struct mlx5_priv *priv = dev->data->dev_private;
9346                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9347                 uint32_t shl_c0 = rte_bsf32(msk_c0);
9348
9349                 mask &= msk_c0;
9350                 mask <<= shl_c0;
9351                 value <<= shl_c0;
9352         }
9353         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
9354 }
9355
9356 /**
9357  * Add TAG item to matcher
9358  *
9359  * @param[in] dev
9360  *   The devich to configure through.
9361  * @param[in, out] matcher
9362  *   Flow matcher.
9363  * @param[in, out] key
9364  *   Flow matcher value.
9365  * @param[in] item
9366  *   Flow pattern to translate.
9367  */
9368 static void
9369 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
9370                            void *matcher, void *key,
9371                            const struct rte_flow_item *item)
9372 {
9373         const struct rte_flow_item_tag *tag_v = item->spec;
9374         const struct rte_flow_item_tag *tag_m = item->mask;
9375         enum modify_reg reg;
9376
9377         MLX5_ASSERT(tag_v);
9378         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
9379         /* Get the metadata register index for the tag. */
9380         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
9381         MLX5_ASSERT(reg > 0);
9382         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
9383 }
9384
9385 /**
9386  * Add source vport match to the specified matcher.
9387  *
9388  * @param[in, out] matcher
9389  *   Flow matcher.
9390  * @param[in, out] key
9391  *   Flow matcher value.
9392  * @param[in] port
9393  *   Source vport value to match
9394  * @param[in] mask
9395  *   Mask
9396  */
9397 static void
9398 flow_dv_translate_item_source_vport(void *matcher, void *key,
9399                                     int16_t port, uint16_t mask)
9400 {
9401         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9402         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9403
9404         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
9405         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
9406 }
9407
9408 /**
9409  * Translate port-id item to eswitch match on  port-id.
9410  *
9411  * @param[in] dev
9412  *   The devich to configure through.
9413  * @param[in, out] matcher
9414  *   Flow matcher.
9415  * @param[in, out] key
9416  *   Flow matcher value.
9417  * @param[in] item
9418  *   Flow pattern to translate.
9419  * @param[in]
9420  *   Flow attributes.
9421  *
9422  * @return
9423  *   0 on success, a negative errno value otherwise.
9424  */
9425 static int
9426 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
9427                                void *key, const struct rte_flow_item *item,
9428                                const struct rte_flow_attr *attr)
9429 {
9430         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
9431         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
9432         struct mlx5_priv *priv;
9433         uint16_t mask, id;
9434
9435         mask = pid_m ? pid_m->id : 0xffff;
9436         id = pid_v ? pid_v->id : dev->data->port_id;
9437         priv = mlx5_port_to_eswitch_info(id, item == NULL);
9438         if (!priv)
9439                 return -rte_errno;
9440         /*
9441          * Translate to vport field or to metadata, depending on mode.
9442          * Kernel can use either misc.source_port or half of C0 metadata
9443          * register.
9444          */
9445         if (priv->vport_meta_mask) {
9446                 /*
9447                  * Provide the hint for SW steering library
9448                  * to insert the flow into ingress domain and
9449                  * save the extra vport match.
9450                  */
9451                 if (mask == 0xffff && priv->vport_id == 0xffff &&
9452                     priv->pf_bond < 0 && attr->transfer)
9453                         flow_dv_translate_item_source_vport
9454                                 (matcher, key, priv->vport_id, mask);
9455                 /*
9456                  * We should always set the vport metadata register,
9457                  * otherwise the SW steering library can drop
9458                  * the rule if wire vport metadata value is not zero,
9459                  * it depends on kernel configuration.
9460                  */
9461                 flow_dv_translate_item_meta_vport(matcher, key,
9462                                                   priv->vport_meta_tag,
9463                                                   priv->vport_meta_mask);
9464         } else {
9465                 flow_dv_translate_item_source_vport(matcher, key,
9466                                                     priv->vport_id, mask);
9467         }
9468         return 0;
9469 }
9470
9471 /**
9472  * Add ICMP6 item to matcher and to the value.
9473  *
9474  * @param[in, out] matcher
9475  *   Flow matcher.
9476  * @param[in, out] key
9477  *   Flow matcher value.
9478  * @param[in] item
9479  *   Flow pattern to translate.
9480  * @param[in] inner
9481  *   Item is inner pattern.
9482  */
9483 static void
9484 flow_dv_translate_item_icmp6(void *matcher, void *key,
9485                               const struct rte_flow_item *item,
9486                               int inner)
9487 {
9488         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
9489         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
9490         void *headers_m;
9491         void *headers_v;
9492         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9493                                      misc_parameters_3);
9494         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9495         if (inner) {
9496                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9497                                          inner_headers);
9498                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9499         } else {
9500                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9501                                          outer_headers);
9502                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9503         }
9504         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9505         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
9506         if (!icmp6_v)
9507                 return;
9508         if (!icmp6_m)
9509                 icmp6_m = &rte_flow_item_icmp6_mask;
9510         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
9511         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
9512                  icmp6_v->type & icmp6_m->type);
9513         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
9514         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
9515                  icmp6_v->code & icmp6_m->code);
9516 }
9517
9518 /**
9519  * Add ICMP item to matcher and to the value.
9520  *
9521  * @param[in, out] matcher
9522  *   Flow matcher.
9523  * @param[in, out] key
9524  *   Flow matcher value.
9525  * @param[in] item
9526  *   Flow pattern to translate.
9527  * @param[in] inner
9528  *   Item is inner pattern.
9529  */
9530 static void
9531 flow_dv_translate_item_icmp(void *matcher, void *key,
9532                             const struct rte_flow_item *item,
9533                             int inner)
9534 {
9535         const struct rte_flow_item_icmp *icmp_m = item->mask;
9536         const struct rte_flow_item_icmp *icmp_v = item->spec;
9537         uint32_t icmp_header_data_m = 0;
9538         uint32_t icmp_header_data_v = 0;
9539         void *headers_m;
9540         void *headers_v;
9541         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9542                                      misc_parameters_3);
9543         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9544         if (inner) {
9545                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9546                                          inner_headers);
9547                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9548         } else {
9549                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9550                                          outer_headers);
9551                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9552         }
9553         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9554         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
9555         if (!icmp_v)
9556                 return;
9557         if (!icmp_m)
9558                 icmp_m = &rte_flow_item_icmp_mask;
9559         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
9560                  icmp_m->hdr.icmp_type);
9561         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
9562                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
9563         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
9564                  icmp_m->hdr.icmp_code);
9565         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
9566                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
9567         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
9568         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
9569         if (icmp_header_data_m) {
9570                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
9571                 icmp_header_data_v |=
9572                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
9573                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
9574                          icmp_header_data_m);
9575                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
9576                          icmp_header_data_v & icmp_header_data_m);
9577         }
9578 }
9579
9580 /**
9581  * Add GTP item to matcher and to the value.
9582  *
9583  * @param[in, out] matcher
9584  *   Flow matcher.
9585  * @param[in, out] key
9586  *   Flow matcher value.
9587  * @param[in] item
9588  *   Flow pattern to translate.
9589  * @param[in] inner
9590  *   Item is inner pattern.
9591  */
9592 static void
9593 flow_dv_translate_item_gtp(void *matcher, void *key,
9594                            const struct rte_flow_item *item, int inner)
9595 {
9596         const struct rte_flow_item_gtp *gtp_m = item->mask;
9597         const struct rte_flow_item_gtp *gtp_v = item->spec;
9598         void *headers_m;
9599         void *headers_v;
9600         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9601                                      misc_parameters_3);
9602         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9603         uint16_t dport = RTE_GTPU_UDP_PORT;
9604
9605         if (inner) {
9606                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9607                                          inner_headers);
9608                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9609         } else {
9610                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9611                                          outer_headers);
9612                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9613         }
9614         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9615                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9616                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9617         }
9618         if (!gtp_v)
9619                 return;
9620         if (!gtp_m)
9621                 gtp_m = &rte_flow_item_gtp_mask;
9622         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
9623                  gtp_m->v_pt_rsv_flags);
9624         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
9625                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
9626         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
9627         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
9628                  gtp_v->msg_type & gtp_m->msg_type);
9629         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
9630                  rte_be_to_cpu_32(gtp_m->teid));
9631         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
9632                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
9633 }
9634
9635 /**
9636  * Add GTP PSC item to matcher.
9637  *
9638  * @param[in, out] matcher
9639  *   Flow matcher.
9640  * @param[in, out] key
9641  *   Flow matcher value.
9642  * @param[in] item
9643  *   Flow pattern to translate.
9644  */
9645 static int
9646 flow_dv_translate_item_gtp_psc(void *matcher, void *key,
9647                                const struct rte_flow_item *item)
9648 {
9649         const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask;
9650         const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec;
9651         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9652                         misc_parameters_3);
9653         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9654         union {
9655                 uint32_t w32;
9656                 struct {
9657                         uint16_t seq_num;
9658                         uint8_t npdu_num;
9659                         uint8_t next_ext_header_type;
9660                 };
9661         } dw_2;
9662         uint8_t gtp_flags;
9663
9664         /* Always set E-flag match on one, regardless of GTP item settings. */
9665         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags);
9666         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9667         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags);
9668         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags);
9669         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9670         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags);
9671         /*Set next extension header type. */
9672         dw_2.seq_num = 0;
9673         dw_2.npdu_num = 0;
9674         dw_2.next_ext_header_type = 0xff;
9675         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2,
9676                  rte_cpu_to_be_32(dw_2.w32));
9677         dw_2.seq_num = 0;
9678         dw_2.npdu_num = 0;
9679         dw_2.next_ext_header_type = 0x85;
9680         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2,
9681                  rte_cpu_to_be_32(dw_2.w32));
9682         if (gtp_psc_v) {
9683                 union {
9684                         uint32_t w32;
9685                         struct {
9686                                 uint8_t len;
9687                                 uint8_t type_flags;
9688                                 uint8_t qfi;
9689                                 uint8_t reserved;
9690                         };
9691                 } dw_0;
9692
9693                 /*Set extension header PDU type and Qos. */
9694                 if (!gtp_psc_m)
9695                         gtp_psc_m = &rte_flow_item_gtp_psc_mask;
9696                 dw_0.w32 = 0;
9697                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->pdu_type);
9698                 dw_0.qfi = gtp_psc_m->qfi;
9699                 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0,
9700                          rte_cpu_to_be_32(dw_0.w32));
9701                 dw_0.w32 = 0;
9702                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->pdu_type &
9703                                                         gtp_psc_m->pdu_type);
9704                 dw_0.qfi = gtp_psc_v->qfi & gtp_psc_m->qfi;
9705                 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0,
9706                          rte_cpu_to_be_32(dw_0.w32));
9707         }
9708         return 0;
9709 }
9710
9711 /**
9712  * Add eCPRI item to matcher and to the value.
9713  *
9714  * @param[in] dev
9715  *   The devich to configure through.
9716  * @param[in, out] matcher
9717  *   Flow matcher.
9718  * @param[in, out] key
9719  *   Flow matcher value.
9720  * @param[in] item
9721  *   Flow pattern to translate.
9722  * @param[in] samples
9723  *   Sample IDs to be used in the matching.
9724  */
9725 static void
9726 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
9727                              void *key, const struct rte_flow_item *item)
9728 {
9729         struct mlx5_priv *priv = dev->data->dev_private;
9730         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
9731         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
9732         struct rte_ecpri_common_hdr common;
9733         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
9734                                      misc_parameters_4);
9735         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
9736         uint32_t *samples;
9737         void *dw_m;
9738         void *dw_v;
9739
9740         if (!ecpri_v)
9741                 return;
9742         if (!ecpri_m)
9743                 ecpri_m = &rte_flow_item_ecpri_mask;
9744         /*
9745          * Maximal four DW samples are supported in a single matching now.
9746          * Two are used now for a eCPRI matching:
9747          * 1. Type: one byte, mask should be 0x00ff0000 in network order
9748          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
9749          *    if any.
9750          */
9751         if (!ecpri_m->hdr.common.u32)
9752                 return;
9753         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
9754         /* Need to take the whole DW as the mask to fill the entry. */
9755         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9756                             prog_sample_field_value_0);
9757         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9758                             prog_sample_field_value_0);
9759         /* Already big endian (network order) in the header. */
9760         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
9761         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32;
9762         /* Sample#0, used for matching type, offset 0. */
9763         MLX5_SET(fte_match_set_misc4, misc4_m,
9764                  prog_sample_field_id_0, samples[0]);
9765         /* It makes no sense to set the sample ID in the mask field. */
9766         MLX5_SET(fte_match_set_misc4, misc4_v,
9767                  prog_sample_field_id_0, samples[0]);
9768         /*
9769          * Checking if message body part needs to be matched.
9770          * Some wildcard rules only matching type field should be supported.
9771          */
9772         if (ecpri_m->hdr.dummy[0]) {
9773                 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32);
9774                 switch (common.type) {
9775                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
9776                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
9777                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
9778                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9779                                             prog_sample_field_value_1);
9780                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9781                                             prog_sample_field_value_1);
9782                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
9783                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] &
9784                                             ecpri_m->hdr.dummy[0];
9785                         /* Sample#1, to match message body, offset 4. */
9786                         MLX5_SET(fte_match_set_misc4, misc4_m,
9787                                  prog_sample_field_id_1, samples[1]);
9788                         MLX5_SET(fte_match_set_misc4, misc4_v,
9789                                  prog_sample_field_id_1, samples[1]);
9790                         break;
9791                 default:
9792                         /* Others, do not match any sample ID. */
9793                         break;
9794                 }
9795         }
9796 }
9797
9798 /*
9799  * Add connection tracking status item to matcher
9800  *
9801  * @param[in] dev
9802  *   The devich to configure through.
9803  * @param[in, out] matcher
9804  *   Flow matcher.
9805  * @param[in, out] key
9806  *   Flow matcher value.
9807  * @param[in] item
9808  *   Flow pattern to translate.
9809  */
9810 static void
9811 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev,
9812                               void *matcher, void *key,
9813                               const struct rte_flow_item *item)
9814 {
9815         uint32_t reg_value = 0;
9816         int reg_id;
9817         /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */
9818         uint32_t reg_mask = 0;
9819         const struct rte_flow_item_conntrack *spec = item->spec;
9820         const struct rte_flow_item_conntrack *mask = item->mask;
9821         uint32_t flags;
9822         struct rte_flow_error error;
9823
9824         if (!mask)
9825                 mask = &rte_flow_item_conntrack_mask;
9826         if (!spec || !mask->flags)
9827                 return;
9828         flags = spec->flags & mask->flags;
9829         /* The conflict should be checked in the validation. */
9830         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID)
9831                 reg_value |= MLX5_CT_SYNDROME_VALID;
9832         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
9833                 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE;
9834         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID)
9835                 reg_value |= MLX5_CT_SYNDROME_INVALID;
9836         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)
9837                 reg_value |= MLX5_CT_SYNDROME_TRAP;
9838         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
9839                 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET;
9840         if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID |
9841                            RTE_FLOW_CONNTRACK_PKT_STATE_INVALID |
9842                            RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))
9843                 reg_mask |= 0xc0;
9844         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
9845                 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE;
9846         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
9847                 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET;
9848         /* The REG_C_x value could be saved during startup. */
9849         reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error);
9850         if (reg_id == REG_NON)
9851                 return;
9852         flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id,
9853                                reg_value, reg_mask);
9854 }
9855
9856 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
9857
9858 #define HEADER_IS_ZERO(match_criteria, headers)                              \
9859         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
9860                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
9861
9862 /**
9863  * Calculate flow matcher enable bitmap.
9864  *
9865  * @param match_criteria
9866  *   Pointer to flow matcher criteria.
9867  *
9868  * @return
9869  *   Bitmap of enabled fields.
9870  */
9871 static uint8_t
9872 flow_dv_matcher_enable(uint32_t *match_criteria)
9873 {
9874         uint8_t match_criteria_enable;
9875
9876         match_criteria_enable =
9877                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
9878                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
9879         match_criteria_enable |=
9880                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
9881                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
9882         match_criteria_enable |=
9883                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
9884                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
9885         match_criteria_enable |=
9886                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
9887                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9888         match_criteria_enable |=
9889                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
9890                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
9891         match_criteria_enable |=
9892                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
9893                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
9894         return match_criteria_enable;
9895 }
9896
9897 struct mlx5_hlist_entry *
9898 flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
9899 {
9900         struct mlx5_dev_ctx_shared *sh = list->ctx;
9901         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
9902         struct rte_eth_dev *dev = ctx->dev;
9903         struct mlx5_flow_tbl_data_entry *tbl_data;
9904         struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data;
9905         struct rte_flow_error *error = ctx->error;
9906         union mlx5_flow_tbl_key key = { .v64 = key64 };
9907         struct mlx5_flow_tbl_resource *tbl;
9908         void *domain;
9909         uint32_t idx = 0;
9910         int ret;
9911
9912         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
9913         if (!tbl_data) {
9914                 rte_flow_error_set(error, ENOMEM,
9915                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9916                                    NULL,
9917                                    "cannot allocate flow table data entry");
9918                 return NULL;
9919         }
9920         tbl_data->idx = idx;
9921         tbl_data->tunnel = tt_prm->tunnel;
9922         tbl_data->group_id = tt_prm->group_id;
9923         tbl_data->external = !!tt_prm->external;
9924         tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
9925         tbl_data->is_egress = !!key.is_egress;
9926         tbl_data->is_transfer = !!key.is_fdb;
9927         tbl_data->dummy = !!key.dummy;
9928         tbl_data->level = key.level;
9929         tbl_data->id = key.id;
9930         tbl = &tbl_data->tbl;
9931         if (key.dummy)
9932                 return &tbl_data->entry;
9933         if (key.is_fdb)
9934                 domain = sh->fdb_domain;
9935         else if (key.is_egress)
9936                 domain = sh->tx_domain;
9937         else
9938                 domain = sh->rx_domain;
9939         ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
9940         if (ret) {
9941                 rte_flow_error_set(error, ENOMEM,
9942                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9943                                    NULL, "cannot create flow table object");
9944                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
9945                 return NULL;
9946         }
9947         if (key.level != 0) {
9948                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
9949                                         (tbl->obj, &tbl_data->jump.action);
9950                 if (ret) {
9951                         rte_flow_error_set(error, ENOMEM,
9952                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9953                                            NULL,
9954                                            "cannot create flow jump action");
9955                         mlx5_flow_os_destroy_flow_tbl(tbl->obj);
9956                         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
9957                         return NULL;
9958                 }
9959         }
9960         MKSTR(matcher_name, "%s_%s_%u_%u_matcher_cache",
9961               key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
9962               key.level, key.id);
9963         mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh,
9964                              flow_dv_matcher_create_cb,
9965                              flow_dv_matcher_match_cb,
9966                              flow_dv_matcher_remove_cb);
9967         return &tbl_data->entry;
9968 }
9969
9970 int
9971 flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
9972                      struct mlx5_hlist_entry *entry, uint64_t key64,
9973                      void *cb_ctx __rte_unused)
9974 {
9975         struct mlx5_flow_tbl_data_entry *tbl_data =
9976                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
9977         union mlx5_flow_tbl_key key = { .v64 = key64 };
9978
9979         return tbl_data->level != key.level ||
9980                tbl_data->id != key.id ||
9981                tbl_data->dummy != key.dummy ||
9982                tbl_data->is_transfer != !!key.is_fdb ||
9983                tbl_data->is_egress != !!key.is_egress;
9984 }
9985
9986 /**
9987  * Get a flow table.
9988  *
9989  * @param[in, out] dev
9990  *   Pointer to rte_eth_dev structure.
9991  * @param[in] table_level
9992  *   Table level to use.
9993  * @param[in] egress
9994  *   Direction of the table.
9995  * @param[in] transfer
9996  *   E-Switch or NIC flow.
9997  * @param[in] dummy
9998  *   Dummy entry for dv API.
9999  * @param[in] table_id
10000  *   Table id to use.
10001  * @param[out] error
10002  *   pointer to error structure.
10003  *
10004  * @return
10005  *   Returns tables resource based on the index, NULL in case of failed.
10006  */
10007 struct mlx5_flow_tbl_resource *
10008 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
10009                          uint32_t table_level, uint8_t egress,
10010                          uint8_t transfer,
10011                          bool external,
10012                          const struct mlx5_flow_tunnel *tunnel,
10013                          uint32_t group_id, uint8_t dummy,
10014                          uint32_t table_id,
10015                          struct rte_flow_error *error)
10016 {
10017         struct mlx5_priv *priv = dev->data->dev_private;
10018         union mlx5_flow_tbl_key table_key = {
10019                 {
10020                         .level = table_level,
10021                         .id = table_id,
10022                         .reserved = 0,
10023                         .dummy = !!dummy,
10024                         .is_fdb = !!transfer,
10025                         .is_egress = !!egress,
10026                 }
10027         };
10028         struct mlx5_flow_tbl_tunnel_prm tt_prm = {
10029                 .tunnel = tunnel,
10030                 .group_id = group_id,
10031                 .external = external,
10032         };
10033         struct mlx5_flow_cb_ctx ctx = {
10034                 .dev = dev,
10035                 .error = error,
10036                 .data = &tt_prm,
10037         };
10038         struct mlx5_hlist_entry *entry;
10039         struct mlx5_flow_tbl_data_entry *tbl_data;
10040
10041         entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
10042         if (!entry) {
10043                 rte_flow_error_set(error, ENOMEM,
10044                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10045                                    "cannot get table");
10046                 return NULL;
10047         }
10048         DRV_LOG(DEBUG, "table_level %u table_id %u "
10049                 "tunnel %u group %u registered.",
10050                 table_level, table_id,
10051                 tunnel ? tunnel->tunnel_id : 0, group_id);
10052         tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10053         return &tbl_data->tbl;
10054 }
10055
10056 void
10057 flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
10058                       struct mlx5_hlist_entry *entry)
10059 {
10060         struct mlx5_dev_ctx_shared *sh = list->ctx;
10061         struct mlx5_flow_tbl_data_entry *tbl_data =
10062                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10063
10064         MLX5_ASSERT(entry && sh);
10065         if (tbl_data->jump.action)
10066                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10067         if (tbl_data->tbl.obj)
10068                 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
10069         if (tbl_data->tunnel_offload && tbl_data->external) {
10070                 struct mlx5_hlist_entry *he;
10071                 struct mlx5_hlist *tunnel_grp_hash;
10072                 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10073                 union tunnel_tbl_key tunnel_key = {
10074                         .tunnel_id = tbl_data->tunnel ?
10075                                         tbl_data->tunnel->tunnel_id : 0,
10076                         .group = tbl_data->group_id
10077                 };
10078                 uint32_t table_level = tbl_data->level;
10079
10080                 tunnel_grp_hash = tbl_data->tunnel ?
10081                                         tbl_data->tunnel->groups :
10082                                         thub->groups;
10083                 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
10084                 if (he)
10085                         mlx5_hlist_unregister(tunnel_grp_hash, he);
10086                 DRV_LOG(DEBUG,
10087                         "table_level %u id %u tunnel %u group %u released.",
10088                         table_level,
10089                         tbl_data->id,
10090                         tbl_data->tunnel ?
10091                         tbl_data->tunnel->tunnel_id : 0,
10092                         tbl_data->group_id);
10093         }
10094         mlx5_cache_list_destroy(&tbl_data->matchers);
10095         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10096 }
10097
10098 /**
10099  * Release a flow table.
10100  *
10101  * @param[in] sh
10102  *   Pointer to device shared structure.
10103  * @param[in] tbl
10104  *   Table resource to be released.
10105  *
10106  * @return
10107  *   Returns 0 if table was released, else return 1;
10108  */
10109 static int
10110 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
10111                              struct mlx5_flow_tbl_resource *tbl)
10112 {
10113         struct mlx5_flow_tbl_data_entry *tbl_data =
10114                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10115
10116         if (!tbl)
10117                 return 0;
10118         return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
10119 }
10120
10121 int
10122 flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
10123                          struct mlx5_cache_entry *entry, void *cb_ctx)
10124 {
10125         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10126         struct mlx5_flow_dv_matcher *ref = ctx->data;
10127         struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur),
10128                                                         entry);
10129
10130         return cur->crc != ref->crc ||
10131                cur->priority != ref->priority ||
10132                memcmp((const void *)cur->mask.buf,
10133                       (const void *)ref->mask.buf, ref->mask.size);
10134 }
10135
10136 struct mlx5_cache_entry *
10137 flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
10138                           struct mlx5_cache_entry *entry __rte_unused,
10139                           void *cb_ctx)
10140 {
10141         struct mlx5_dev_ctx_shared *sh = list->ctx;
10142         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10143         struct mlx5_flow_dv_matcher *ref = ctx->data;
10144         struct mlx5_flow_dv_matcher *cache;
10145         struct mlx5dv_flow_matcher_attr dv_attr = {
10146                 .type = IBV_FLOW_ATTR_NORMAL,
10147                 .match_mask = (void *)&ref->mask,
10148         };
10149         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10150                                                             typeof(*tbl), tbl);
10151         int ret;
10152
10153         cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY);
10154         if (!cache) {
10155                 rte_flow_error_set(ctx->error, ENOMEM,
10156                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10157                                    "cannot create matcher");
10158                 return NULL;
10159         }
10160         *cache = *ref;
10161         dv_attr.match_criteria_enable =
10162                 flow_dv_matcher_enable(cache->mask.buf);
10163         dv_attr.priority = ref->priority;
10164         if (tbl->is_egress)
10165                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
10166         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
10167                                                &cache->matcher_object);
10168         if (ret) {
10169                 mlx5_free(cache);
10170                 rte_flow_error_set(ctx->error, ENOMEM,
10171                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10172                                    "cannot create matcher");
10173                 return NULL;
10174         }
10175         return &cache->entry;
10176 }
10177
10178 /**
10179  * Register the flow matcher.
10180  *
10181  * @param[in, out] dev
10182  *   Pointer to rte_eth_dev structure.
10183  * @param[in, out] matcher
10184  *   Pointer to flow matcher.
10185  * @param[in, out] key
10186  *   Pointer to flow table key.
10187  * @parm[in, out] dev_flow
10188  *   Pointer to the dev_flow.
10189  * @param[out] error
10190  *   pointer to error structure.
10191  *
10192  * @return
10193  *   0 on success otherwise -errno and errno is set.
10194  */
10195 static int
10196 flow_dv_matcher_register(struct rte_eth_dev *dev,
10197                          struct mlx5_flow_dv_matcher *ref,
10198                          union mlx5_flow_tbl_key *key,
10199                          struct mlx5_flow *dev_flow,
10200                          const struct mlx5_flow_tunnel *tunnel,
10201                          uint32_t group_id,
10202                          struct rte_flow_error *error)
10203 {
10204         struct mlx5_cache_entry *entry;
10205         struct mlx5_flow_dv_matcher *cache;
10206         struct mlx5_flow_tbl_resource *tbl;
10207         struct mlx5_flow_tbl_data_entry *tbl_data;
10208         struct mlx5_flow_cb_ctx ctx = {
10209                 .error = error,
10210                 .data = ref,
10211         };
10212
10213         /**
10214          * tunnel offload API requires this registration for cases when
10215          * tunnel match rule was inserted before tunnel set rule.
10216          */
10217         tbl = flow_dv_tbl_resource_get(dev, key->level,
10218                                        key->is_egress, key->is_fdb,
10219                                        dev_flow->external, tunnel,
10220                                        group_id, 0, key->id, error);
10221         if (!tbl)
10222                 return -rte_errno;      /* No need to refill the error info */
10223         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10224         ref->tbl = tbl;
10225         entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
10226         if (!entry) {
10227                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
10228                 return rte_flow_error_set(error, ENOMEM,
10229                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10230                                           "cannot allocate ref memory");
10231         }
10232         cache = container_of(entry, typeof(*cache), entry);
10233         dev_flow->handle->dvh.matcher = cache;
10234         return 0;
10235 }
10236
10237 struct mlx5_hlist_entry *
10238 flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
10239 {
10240         struct mlx5_dev_ctx_shared *sh = list->ctx;
10241         struct rte_flow_error *error = ctx;
10242         struct mlx5_flow_dv_tag_resource *entry;
10243         uint32_t idx = 0;
10244         int ret;
10245
10246         entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10247         if (!entry) {
10248                 rte_flow_error_set(error, ENOMEM,
10249                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10250                                    "cannot allocate resource memory");
10251                 return NULL;
10252         }
10253         entry->idx = idx;
10254         entry->tag_id = key;
10255         ret = mlx5_flow_os_create_flow_action_tag(key,
10256                                                   &entry->action);
10257         if (ret) {
10258                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
10259                 rte_flow_error_set(error, ENOMEM,
10260                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10261                                    NULL, "cannot create action");
10262                 return NULL;
10263         }
10264         return &entry->entry;
10265 }
10266
10267 int
10268 flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused,
10269                      struct mlx5_hlist_entry *entry, uint64_t key,
10270                      void *cb_ctx __rte_unused)
10271 {
10272         struct mlx5_flow_dv_tag_resource *tag =
10273                 container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10274
10275         return key != tag->tag_id;
10276 }
10277
10278 /**
10279  * Find existing tag resource or create and register a new one.
10280  *
10281  * @param dev[in, out]
10282  *   Pointer to rte_eth_dev structure.
10283  * @param[in, out] tag_be24
10284  *   Tag value in big endian then R-shift 8.
10285  * @parm[in, out] dev_flow
10286  *   Pointer to the dev_flow.
10287  * @param[out] error
10288  *   pointer to error structure.
10289  *
10290  * @return
10291  *   0 on success otherwise -errno and errno is set.
10292  */
10293 static int
10294 flow_dv_tag_resource_register
10295                         (struct rte_eth_dev *dev,
10296                          uint32_t tag_be24,
10297                          struct mlx5_flow *dev_flow,
10298                          struct rte_flow_error *error)
10299 {
10300         struct mlx5_priv *priv = dev->data->dev_private;
10301         struct mlx5_flow_dv_tag_resource *cache_resource;
10302         struct mlx5_hlist_entry *entry;
10303
10304         entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
10305         if (entry) {
10306                 cache_resource = container_of
10307                         (entry, struct mlx5_flow_dv_tag_resource, entry);
10308                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
10309                 dev_flow->dv.tag_resource = cache_resource;
10310                 return 0;
10311         }
10312         return -rte_errno;
10313 }
10314
10315 void
10316 flow_dv_tag_remove_cb(struct mlx5_hlist *list,
10317                       struct mlx5_hlist_entry *entry)
10318 {
10319         struct mlx5_dev_ctx_shared *sh = list->ctx;
10320         struct mlx5_flow_dv_tag_resource *tag =
10321                 container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10322
10323         MLX5_ASSERT(tag && sh && tag->action);
10324         claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
10325         DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag);
10326         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10327 }
10328
10329 /**
10330  * Release the tag.
10331  *
10332  * @param dev
10333  *   Pointer to Ethernet device.
10334  * @param tag_idx
10335  *   Tag index.
10336  *
10337  * @return
10338  *   1 while a reference on it exists, 0 when freed.
10339  */
10340 static int
10341 flow_dv_tag_release(struct rte_eth_dev *dev,
10342                     uint32_t tag_idx)
10343 {
10344         struct mlx5_priv *priv = dev->data->dev_private;
10345         struct mlx5_flow_dv_tag_resource *tag;
10346
10347         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
10348         if (!tag)
10349                 return 0;
10350         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
10351                 dev->data->port_id, (void *)tag, tag->entry.ref_cnt);
10352         return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry);
10353 }
10354
10355 /**
10356  * Translate port ID action to vport.
10357  *
10358  * @param[in] dev
10359  *   Pointer to rte_eth_dev structure.
10360  * @param[in] action
10361  *   Pointer to the port ID action.
10362  * @param[out] dst_port_id
10363  *   The target port ID.
10364  * @param[out] error
10365  *   Pointer to the error structure.
10366  *
10367  * @return
10368  *   0 on success, a negative errno value otherwise and rte_errno is set.
10369  */
10370 static int
10371 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
10372                                  const struct rte_flow_action *action,
10373                                  uint32_t *dst_port_id,
10374                                  struct rte_flow_error *error)
10375 {
10376         uint32_t port;
10377         struct mlx5_priv *priv;
10378         const struct rte_flow_action_port_id *conf =
10379                         (const struct rte_flow_action_port_id *)action->conf;
10380
10381         port = conf->original ? dev->data->port_id : conf->id;
10382         priv = mlx5_port_to_eswitch_info(port, false);
10383         if (!priv)
10384                 return rte_flow_error_set(error, -rte_errno,
10385                                           RTE_FLOW_ERROR_TYPE_ACTION,
10386                                           NULL,
10387                                           "No eswitch info was found for port");
10388 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
10389         /*
10390          * This parameter is transferred to
10391          * mlx5dv_dr_action_create_dest_ib_port().
10392          */
10393         *dst_port_id = priv->dev_port;
10394 #else
10395         /*
10396          * Legacy mode, no LAG configurations is supported.
10397          * This parameter is transferred to
10398          * mlx5dv_dr_action_create_dest_vport().
10399          */
10400         *dst_port_id = priv->vport_id;
10401 #endif
10402         return 0;
10403 }
10404
10405 /**
10406  * Create a counter with aging configuration.
10407  *
10408  * @param[in] dev
10409  *   Pointer to rte_eth_dev structure.
10410  * @param[in] dev_flow
10411  *   Pointer to the mlx5_flow.
10412  * @param[out] count
10413  *   Pointer to the counter action configuration.
10414  * @param[in] age
10415  *   Pointer to the aging action configuration.
10416  *
10417  * @return
10418  *   Index to flow counter on success, 0 otherwise.
10419  */
10420 static uint32_t
10421 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
10422                                 struct mlx5_flow *dev_flow,
10423                                 const struct rte_flow_action_count *count,
10424                                 const struct rte_flow_action_age *age)
10425 {
10426         uint32_t counter;
10427         struct mlx5_age_param *age_param;
10428
10429         if (count && count->shared)
10430                 counter = flow_dv_counter_get_shared(dev, count->id);
10431         else
10432                 counter = flow_dv_counter_alloc(dev, !!age);
10433         if (!counter || age == NULL)
10434                 return counter;
10435         age_param = flow_dv_counter_idx_get_age(dev, counter);
10436         age_param->context = age->context ? age->context :
10437                 (void *)(uintptr_t)(dev_flow->flow_idx);
10438         age_param->timeout = age->timeout;
10439         age_param->port_id = dev->data->port_id;
10440         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
10441         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
10442         return counter;
10443 }
10444
10445 /**
10446  * Add Tx queue matcher
10447  *
10448  * @param[in] dev
10449  *   Pointer to the dev struct.
10450  * @param[in, out] matcher
10451  *   Flow matcher.
10452  * @param[in, out] key
10453  *   Flow matcher value.
10454  * @param[in] item
10455  *   Flow pattern to translate.
10456  * @param[in] inner
10457  *   Item is inner pattern.
10458  */
10459 static void
10460 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
10461                                 void *matcher, void *key,
10462                                 const struct rte_flow_item *item)
10463 {
10464         const struct mlx5_rte_flow_item_tx_queue *queue_m;
10465         const struct mlx5_rte_flow_item_tx_queue *queue_v;
10466         void *misc_m =
10467                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
10468         void *misc_v =
10469                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
10470         struct mlx5_txq_ctrl *txq;
10471         uint32_t queue;
10472
10473
10474         queue_m = (const void *)item->mask;
10475         if (!queue_m)
10476                 return;
10477         queue_v = (const void *)item->spec;
10478         if (!queue_v)
10479                 return;
10480         txq = mlx5_txq_get(dev, queue_v->queue);
10481         if (!txq)
10482                 return;
10483         queue = txq->obj->sq->id;
10484         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
10485         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
10486                  queue & queue_m->queue);
10487         mlx5_txq_release(dev, queue_v->queue);
10488 }
10489
10490 /**
10491  * Set the hash fields according to the @p flow information.
10492  *
10493  * @param[in] dev_flow
10494  *   Pointer to the mlx5_flow.
10495  * @param[in] rss_desc
10496  *   Pointer to the mlx5_flow_rss_desc.
10497  */
10498 static void
10499 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
10500                        struct mlx5_flow_rss_desc *rss_desc)
10501 {
10502         uint64_t items = dev_flow->handle->layers;
10503         int rss_inner = 0;
10504         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
10505
10506         dev_flow->hash_fields = 0;
10507 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
10508         if (rss_desc->level >= 2) {
10509                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
10510                 rss_inner = 1;
10511         }
10512 #endif
10513         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
10514             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
10515                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
10516                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10517                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
10518                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10519                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
10520                         else
10521                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
10522                 }
10523         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
10524                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
10525                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
10526                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10527                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
10528                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10529                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
10530                         else
10531                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
10532                 }
10533         }
10534         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
10535             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
10536                 if (rss_types & ETH_RSS_UDP) {
10537                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10538                                 dev_flow->hash_fields |=
10539                                                 IBV_RX_HASH_SRC_PORT_UDP;
10540                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10541                                 dev_flow->hash_fields |=
10542                                                 IBV_RX_HASH_DST_PORT_UDP;
10543                         else
10544                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
10545                 }
10546         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
10547                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
10548                 if (rss_types & ETH_RSS_TCP) {
10549                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10550                                 dev_flow->hash_fields |=
10551                                                 IBV_RX_HASH_SRC_PORT_TCP;
10552                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10553                                 dev_flow->hash_fields |=
10554                                                 IBV_RX_HASH_DST_PORT_TCP;
10555                         else
10556                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
10557                 }
10558         }
10559 }
10560
10561 /**
10562  * Prepare an Rx Hash queue.
10563  *
10564  * @param dev
10565  *   Pointer to Ethernet device.
10566  * @param[in] dev_flow
10567  *   Pointer to the mlx5_flow.
10568  * @param[in] rss_desc
10569  *   Pointer to the mlx5_flow_rss_desc.
10570  * @param[out] hrxq_idx
10571  *   Hash Rx queue index.
10572  *
10573  * @return
10574  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
10575  */
10576 static struct mlx5_hrxq *
10577 flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
10578                      struct mlx5_flow *dev_flow,
10579                      struct mlx5_flow_rss_desc *rss_desc,
10580                      uint32_t *hrxq_idx)
10581 {
10582         struct mlx5_priv *priv = dev->data->dev_private;
10583         struct mlx5_flow_handle *dh = dev_flow->handle;
10584         struct mlx5_hrxq *hrxq;
10585
10586         MLX5_ASSERT(rss_desc->queue_num);
10587         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
10588         rss_desc->hash_fields = dev_flow->hash_fields;
10589         rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL);
10590         rss_desc->shared_rss = 0;
10591         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
10592         if (!*hrxq_idx)
10593                 return NULL;
10594         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10595                               *hrxq_idx);
10596         return hrxq;
10597 }
10598
10599 /**
10600  * Release sample sub action resource.
10601  *
10602  * @param[in, out] dev
10603  *   Pointer to rte_eth_dev structure.
10604  * @param[in] act_res
10605  *   Pointer to sample sub action resource.
10606  */
10607 static void
10608 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
10609                                    struct mlx5_flow_sub_actions_idx *act_res)
10610 {
10611         if (act_res->rix_hrxq) {
10612                 mlx5_hrxq_release(dev, act_res->rix_hrxq);
10613                 act_res->rix_hrxq = 0;
10614         }
10615         if (act_res->rix_encap_decap) {
10616                 flow_dv_encap_decap_resource_release(dev,
10617                                                      act_res->rix_encap_decap);
10618                 act_res->rix_encap_decap = 0;
10619         }
10620         if (act_res->rix_port_id_action) {
10621                 flow_dv_port_id_action_resource_release(dev,
10622                                                 act_res->rix_port_id_action);
10623                 act_res->rix_port_id_action = 0;
10624         }
10625         if (act_res->rix_tag) {
10626                 flow_dv_tag_release(dev, act_res->rix_tag);
10627                 act_res->rix_tag = 0;
10628         }
10629         if (act_res->rix_jump) {
10630                 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump);
10631                 act_res->rix_jump = 0;
10632         }
10633 }
10634
10635 int
10636 flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused,
10637                         struct mlx5_cache_entry *entry, void *cb_ctx)
10638 {
10639         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10640         struct rte_eth_dev *dev = ctx->dev;
10641         struct mlx5_flow_dv_sample_resource *resource = ctx->data;
10642         struct mlx5_flow_dv_sample_resource *cache_resource =
10643                         container_of(entry, typeof(*cache_resource), entry);
10644
10645         if (resource->ratio == cache_resource->ratio &&
10646             resource->ft_type == cache_resource->ft_type &&
10647             resource->ft_id == cache_resource->ft_id &&
10648             resource->set_action == cache_resource->set_action &&
10649             !memcmp((void *)&resource->sample_act,
10650                     (void *)&cache_resource->sample_act,
10651                     sizeof(struct mlx5_flow_sub_actions_list))) {
10652                 /*
10653                  * Existing sample action should release the prepared
10654                  * sub-actions reference counter.
10655                  */
10656                 flow_dv_sample_sub_actions_release(dev,
10657                                                 &resource->sample_idx);
10658                 return 0;
10659         }
10660         return 1;
10661 }
10662
10663 struct mlx5_cache_entry *
10664 flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
10665                          struct mlx5_cache_entry *entry __rte_unused,
10666                          void *cb_ctx)
10667 {
10668         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10669         struct rte_eth_dev *dev = ctx->dev;
10670         struct mlx5_flow_dv_sample_resource *resource = ctx->data;
10671         void **sample_dv_actions = resource->sub_actions;
10672         struct mlx5_flow_dv_sample_resource *cache_resource;
10673         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
10674         struct mlx5_priv *priv = dev->data->dev_private;
10675         struct mlx5_dev_ctx_shared *sh = priv->sh;
10676         struct mlx5_flow_tbl_resource *tbl;
10677         uint32_t idx = 0;
10678         const uint32_t next_ft_step = 1;
10679         uint32_t next_ft_id = resource->ft_id + next_ft_step;
10680         uint8_t is_egress = 0;
10681         uint8_t is_transfer = 0;
10682         struct rte_flow_error *error = ctx->error;
10683
10684         /* Register new sample resource. */
10685         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
10686         if (!cache_resource) {
10687                 rte_flow_error_set(error, ENOMEM,
10688                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10689                                           NULL,
10690                                           "cannot allocate resource memory");
10691                 return NULL;
10692         }
10693         *cache_resource = *resource;
10694         /* Create normal path table level */
10695         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
10696                 is_transfer = 1;
10697         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
10698                 is_egress = 1;
10699         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
10700                                         is_egress, is_transfer,
10701                                         true, NULL, 0, 0, 0, error);
10702         if (!tbl) {
10703                 rte_flow_error_set(error, ENOMEM,
10704                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10705                                           NULL,
10706                                           "fail to create normal path table "
10707                                           "for sample");
10708                 goto error;
10709         }
10710         cache_resource->normal_path_tbl = tbl;
10711         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10712                 if (!sh->default_miss_action) {
10713                         rte_flow_error_set(error, ENOMEM,
10714                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10715                                                 NULL,
10716                                                 "default miss action was not "
10717                                                 "created");
10718                         goto error;
10719                 }
10720                 sample_dv_actions[resource->sample_act.actions_num++] =
10721                                                 sh->default_miss_action;
10722         }
10723         /* Create a DR sample action */
10724         sampler_attr.sample_ratio = cache_resource->ratio;
10725         sampler_attr.default_next_table = tbl->obj;
10726         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
10727         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
10728                                                         &sample_dv_actions[0];
10729         sampler_attr.action = cache_resource->set_action;
10730         if (mlx5_os_flow_dr_create_flow_action_sampler
10731                         (&sampler_attr, &cache_resource->verbs_action)) {
10732                 rte_flow_error_set(error, ENOMEM,
10733                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10734                                         NULL, "cannot create sample action");
10735                 goto error;
10736         }
10737         cache_resource->idx = idx;
10738         cache_resource->dev = dev;
10739         return &cache_resource->entry;
10740 error:
10741         if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
10742                 flow_dv_sample_sub_actions_release(dev,
10743                                                    &cache_resource->sample_idx);
10744         if (cache_resource->normal_path_tbl)
10745                 flow_dv_tbl_resource_release(MLX5_SH(dev),
10746                                 cache_resource->normal_path_tbl);
10747         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10748         return NULL;
10749
10750 }
10751
10752 /**
10753  * Find existing sample resource or create and register a new one.
10754  *
10755  * @param[in, out] dev
10756  *   Pointer to rte_eth_dev structure.
10757  * @param[in] resource
10758  *   Pointer to sample resource.
10759  * @parm[in, out] dev_flow
10760  *   Pointer to the dev_flow.
10761  * @param[out] error
10762  *   pointer to error structure.
10763  *
10764  * @return
10765  *   0 on success otherwise -errno and errno is set.
10766  */
10767 static int
10768 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
10769                          struct mlx5_flow_dv_sample_resource *resource,
10770                          struct mlx5_flow *dev_flow,
10771                          struct rte_flow_error *error)
10772 {
10773         struct mlx5_flow_dv_sample_resource *cache_resource;
10774         struct mlx5_cache_entry *entry;
10775         struct mlx5_priv *priv = dev->data->dev_private;
10776         struct mlx5_flow_cb_ctx ctx = {
10777                 .dev = dev,
10778                 .error = error,
10779                 .data = resource,
10780         };
10781
10782         entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx);
10783         if (!entry)
10784                 return -rte_errno;
10785         cache_resource = container_of(entry, typeof(*cache_resource), entry);
10786         dev_flow->handle->dvh.rix_sample = cache_resource->idx;
10787         dev_flow->dv.sample_res = cache_resource;
10788         return 0;
10789 }
10790
10791 int
10792 flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused,
10793                             struct mlx5_cache_entry *entry, void *cb_ctx)
10794 {
10795         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10796         struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
10797         struct rte_eth_dev *dev = ctx->dev;
10798         struct mlx5_flow_dv_dest_array_resource *cache_resource =
10799                         container_of(entry, typeof(*cache_resource), entry);
10800         uint32_t idx = 0;
10801
10802         if (resource->num_of_dest == cache_resource->num_of_dest &&
10803             resource->ft_type == cache_resource->ft_type &&
10804             !memcmp((void *)cache_resource->sample_act,
10805                     (void *)resource->sample_act,
10806                    (resource->num_of_dest *
10807                    sizeof(struct mlx5_flow_sub_actions_list)))) {
10808                 /*
10809                  * Existing sample action should release the prepared
10810                  * sub-actions reference counter.
10811                  */
10812                 for (idx = 0; idx < resource->num_of_dest; idx++)
10813                         flow_dv_sample_sub_actions_release(dev,
10814                                         &resource->sample_idx[idx]);
10815                 return 0;
10816         }
10817         return 1;
10818 }
10819
10820 struct mlx5_cache_entry *
10821 flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
10822                          struct mlx5_cache_entry *entry __rte_unused,
10823                          void *cb_ctx)
10824 {
10825         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10826         struct rte_eth_dev *dev = ctx->dev;
10827         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10828         struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
10829         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
10830         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
10831         struct mlx5_priv *priv = dev->data->dev_private;
10832         struct mlx5_dev_ctx_shared *sh = priv->sh;
10833         struct mlx5_flow_sub_actions_list *sample_act;
10834         struct mlx5dv_dr_domain *domain;
10835         uint32_t idx = 0, res_idx = 0;
10836         struct rte_flow_error *error = ctx->error;
10837         uint64_t action_flags;
10838         int ret;
10839
10840         /* Register new destination array resource. */
10841         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10842                                             &res_idx);
10843         if (!cache_resource) {
10844                 rte_flow_error_set(error, ENOMEM,
10845                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10846                                           NULL,
10847                                           "cannot allocate resource memory");
10848                 return NULL;
10849         }
10850         *cache_resource = *resource;
10851         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
10852                 domain = sh->fdb_domain;
10853         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
10854                 domain = sh->rx_domain;
10855         else
10856                 domain = sh->tx_domain;
10857         for (idx = 0; idx < resource->num_of_dest; idx++) {
10858                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
10859                                  mlx5_malloc(MLX5_MEM_ZERO,
10860                                  sizeof(struct mlx5dv_dr_action_dest_attr),
10861                                  0, SOCKET_ID_ANY);
10862                 if (!dest_attr[idx]) {
10863                         rte_flow_error_set(error, ENOMEM,
10864                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10865                                            NULL,
10866                                            "cannot allocate resource memory");
10867                         goto error;
10868                 }
10869                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
10870                 sample_act = &resource->sample_act[idx];
10871                 action_flags = sample_act->action_flags;
10872                 switch (action_flags) {
10873                 case MLX5_FLOW_ACTION_QUEUE:
10874                         dest_attr[idx]->dest = sample_act->dr_queue_action;
10875                         break;
10876                 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP):
10877                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
10878                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
10879                         dest_attr[idx]->dest_reformat->reformat =
10880                                         sample_act->dr_encap_action;
10881                         dest_attr[idx]->dest_reformat->dest =
10882                                         sample_act->dr_port_id_action;
10883                         break;
10884                 case MLX5_FLOW_ACTION_PORT_ID:
10885                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
10886                         break;
10887                 case MLX5_FLOW_ACTION_JUMP:
10888                         dest_attr[idx]->dest = sample_act->dr_jump_action;
10889                         break;
10890                 default:
10891                         rte_flow_error_set(error, EINVAL,
10892                                            RTE_FLOW_ERROR_TYPE_ACTION,
10893                                            NULL,
10894                                            "unsupported actions type");
10895                         goto error;
10896                 }
10897         }
10898         /* create a dest array actioin */
10899         ret = mlx5_os_flow_dr_create_flow_action_dest_array
10900                                                 (domain,
10901                                                  cache_resource->num_of_dest,
10902                                                  dest_attr,
10903                                                  &cache_resource->action);
10904         if (ret) {
10905                 rte_flow_error_set(error, ENOMEM,
10906                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10907                                    NULL,
10908                                    "cannot create destination array action");
10909                 goto error;
10910         }
10911         cache_resource->idx = res_idx;
10912         cache_resource->dev = dev;
10913         for (idx = 0; idx < resource->num_of_dest; idx++)
10914                 mlx5_free(dest_attr[idx]);
10915         return &cache_resource->entry;
10916 error:
10917         for (idx = 0; idx < resource->num_of_dest; idx++) {
10918                 flow_dv_sample_sub_actions_release(dev,
10919                                 &cache_resource->sample_idx[idx]);
10920                 if (dest_attr[idx])
10921                         mlx5_free(dest_attr[idx]);
10922         }
10923
10924         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
10925         return NULL;
10926 }
10927
10928 /**
10929  * Find existing destination array resource or create and register a new one.
10930  *
10931  * @param[in, out] dev
10932  *   Pointer to rte_eth_dev structure.
10933  * @param[in] resource
10934  *   Pointer to destination array resource.
10935  * @parm[in, out] dev_flow
10936  *   Pointer to the dev_flow.
10937  * @param[out] error
10938  *   pointer to error structure.
10939  *
10940  * @return
10941  *   0 on success otherwise -errno and errno is set.
10942  */
10943 static int
10944 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
10945                          struct mlx5_flow_dv_dest_array_resource *resource,
10946                          struct mlx5_flow *dev_flow,
10947                          struct rte_flow_error *error)
10948 {
10949         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10950         struct mlx5_priv *priv = dev->data->dev_private;
10951         struct mlx5_cache_entry *entry;
10952         struct mlx5_flow_cb_ctx ctx = {
10953                 .dev = dev,
10954                 .error = error,
10955                 .data = resource,
10956         };
10957
10958         entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx);
10959         if (!entry)
10960                 return -rte_errno;
10961         cache_resource = container_of(entry, typeof(*cache_resource), entry);
10962         dev_flow->handle->dvh.rix_dest_array = cache_resource->idx;
10963         dev_flow->dv.dest_array_res = cache_resource;
10964         return 0;
10965 }
10966
10967 /**
10968  * Convert Sample action to DV specification.
10969  *
10970  * @param[in] dev
10971  *   Pointer to rte_eth_dev structure.
10972  * @param[in] action
10973  *   Pointer to sample action structure.
10974  * @param[in, out] dev_flow
10975  *   Pointer to the mlx5_flow.
10976  * @param[in] attr
10977  *   Pointer to the flow attributes.
10978  * @param[in, out] num_of_dest
10979  *   Pointer to the num of destination.
10980  * @param[in, out] sample_actions
10981  *   Pointer to sample actions list.
10982  * @param[in, out] res
10983  *   Pointer to sample resource.
10984  * @param[out] error
10985  *   Pointer to the error structure.
10986  *
10987  * @return
10988  *   0 on success, a negative errno value otherwise and rte_errno is set.
10989  */
10990 static int
10991 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
10992                                 const struct rte_flow_action_sample *action,
10993                                 struct mlx5_flow *dev_flow,
10994                                 const struct rte_flow_attr *attr,
10995                                 uint32_t *num_of_dest,
10996                                 void **sample_actions,
10997                                 struct mlx5_flow_dv_sample_resource *res,
10998                                 struct rte_flow_error *error)
10999 {
11000         struct mlx5_priv *priv = dev->data->dev_private;
11001         const struct rte_flow_action *sub_actions;
11002         struct mlx5_flow_sub_actions_list *sample_act;
11003         struct mlx5_flow_sub_actions_idx *sample_idx;
11004         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11005         struct rte_flow *flow = dev_flow->flow;
11006         struct mlx5_flow_rss_desc *rss_desc;
11007         uint64_t action_flags = 0;
11008
11009         MLX5_ASSERT(wks);
11010         rss_desc = &wks->rss_desc;
11011         sample_act = &res->sample_act;
11012         sample_idx = &res->sample_idx;
11013         res->ratio = action->ratio;
11014         sub_actions = action->actions;
11015         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
11016                 int type = sub_actions->type;
11017                 uint32_t pre_rix = 0;
11018                 void *pre_r;
11019                 switch (type) {
11020                 case RTE_FLOW_ACTION_TYPE_QUEUE:
11021                 {
11022                         const struct rte_flow_action_queue *queue;
11023                         struct mlx5_hrxq *hrxq;
11024                         uint32_t hrxq_idx;
11025
11026                         queue = sub_actions->conf;
11027                         rss_desc->queue_num = 1;
11028                         rss_desc->queue[0] = queue->index;
11029                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11030                                                     rss_desc, &hrxq_idx);
11031                         if (!hrxq)
11032                                 return rte_flow_error_set
11033                                         (error, rte_errno,
11034                                          RTE_FLOW_ERROR_TYPE_ACTION,
11035                                          NULL,
11036                                          "cannot create fate queue");
11037                         sample_act->dr_queue_action = hrxq->action;
11038                         sample_idx->rix_hrxq = hrxq_idx;
11039                         sample_actions[sample_act->actions_num++] =
11040                                                 hrxq->action;
11041                         (*num_of_dest)++;
11042                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
11043                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11044                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11045                         dev_flow->handle->fate_action =
11046                                         MLX5_FLOW_FATE_QUEUE;
11047                         break;
11048                 }
11049                 case RTE_FLOW_ACTION_TYPE_RSS:
11050                 {
11051                         struct mlx5_hrxq *hrxq;
11052                         uint32_t hrxq_idx;
11053                         const struct rte_flow_action_rss *rss;
11054                         const uint8_t *rss_key;
11055
11056                         rss = sub_actions->conf;
11057                         memcpy(rss_desc->queue, rss->queue,
11058                                rss->queue_num * sizeof(uint16_t));
11059                         rss_desc->queue_num = rss->queue_num;
11060                         /* NULL RSS key indicates default RSS key. */
11061                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11062                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11063                         /*
11064                          * rss->level and rss.types should be set in advance
11065                          * when expanding items for RSS.
11066                          */
11067                         flow_dv_hashfields_set(dev_flow, rss_desc);
11068                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11069                                                     rss_desc, &hrxq_idx);
11070                         if (!hrxq)
11071                                 return rte_flow_error_set
11072                                         (error, rte_errno,
11073                                          RTE_FLOW_ERROR_TYPE_ACTION,
11074                                          NULL,
11075                                          "cannot create fate queue");
11076                         sample_act->dr_queue_action = hrxq->action;
11077                         sample_idx->rix_hrxq = hrxq_idx;
11078                         sample_actions[sample_act->actions_num++] =
11079                                                 hrxq->action;
11080                         (*num_of_dest)++;
11081                         action_flags |= MLX5_FLOW_ACTION_RSS;
11082                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11083                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11084                         dev_flow->handle->fate_action =
11085                                         MLX5_FLOW_FATE_QUEUE;
11086                         break;
11087                 }
11088                 case RTE_FLOW_ACTION_TYPE_MARK:
11089                 {
11090                         uint32_t tag_be = mlx5_flow_mark_set
11091                                 (((const struct rte_flow_action_mark *)
11092                                 (sub_actions->conf))->id);
11093
11094                         dev_flow->handle->mark = 1;
11095                         pre_rix = dev_flow->handle->dvh.rix_tag;
11096                         /* Save the mark resource before sample */
11097                         pre_r = dev_flow->dv.tag_resource;
11098                         if (flow_dv_tag_resource_register(dev, tag_be,
11099                                                   dev_flow, error))
11100                                 return -rte_errno;
11101                         MLX5_ASSERT(dev_flow->dv.tag_resource);
11102                         sample_act->dr_tag_action =
11103                                 dev_flow->dv.tag_resource->action;
11104                         sample_idx->rix_tag =
11105                                 dev_flow->handle->dvh.rix_tag;
11106                         sample_actions[sample_act->actions_num++] =
11107                                                 sample_act->dr_tag_action;
11108                         /* Recover the mark resource after sample */
11109                         dev_flow->dv.tag_resource = pre_r;
11110                         dev_flow->handle->dvh.rix_tag = pre_rix;
11111                         action_flags |= MLX5_FLOW_ACTION_MARK;
11112                         break;
11113                 }
11114                 case RTE_FLOW_ACTION_TYPE_COUNT:
11115                 {
11116                         if (!flow->counter) {
11117                                 flow->counter =
11118                                         flow_dv_translate_create_counter(dev,
11119                                                 dev_flow, sub_actions->conf,
11120                                                 0);
11121                                 if (!flow->counter)
11122                                         return rte_flow_error_set
11123                                                 (error, rte_errno,
11124                                                 RTE_FLOW_ERROR_TYPE_ACTION,
11125                                                 NULL,
11126                                                 "cannot create counter"
11127                                                 " object.");
11128                         }
11129                         sample_act->dr_cnt_action =
11130                                   (flow_dv_counter_get_by_idx(dev,
11131                                   flow->counter, NULL))->action;
11132                         sample_actions[sample_act->actions_num++] =
11133                                                 sample_act->dr_cnt_action;
11134                         action_flags |= MLX5_FLOW_ACTION_COUNT;
11135                         break;
11136                 }
11137                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
11138                 {
11139                         struct mlx5_flow_dv_port_id_action_resource
11140                                         port_id_resource;
11141                         uint32_t port_id = 0;
11142
11143                         memset(&port_id_resource, 0, sizeof(port_id_resource));
11144                         /* Save the port id resource before sample */
11145                         pre_rix = dev_flow->handle->rix_port_id_action;
11146                         pre_r = dev_flow->dv.port_id_action;
11147                         if (flow_dv_translate_action_port_id(dev, sub_actions,
11148                                                              &port_id, error))
11149                                 return -rte_errno;
11150                         port_id_resource.port_id = port_id;
11151                         if (flow_dv_port_id_action_resource_register
11152                             (dev, &port_id_resource, dev_flow, error))
11153                                 return -rte_errno;
11154                         sample_act->dr_port_id_action =
11155                                 dev_flow->dv.port_id_action->action;
11156                         sample_idx->rix_port_id_action =
11157                                 dev_flow->handle->rix_port_id_action;
11158                         sample_actions[sample_act->actions_num++] =
11159                                                 sample_act->dr_port_id_action;
11160                         /* Recover the port id resource after sample */
11161                         dev_flow->dv.port_id_action = pre_r;
11162                         dev_flow->handle->rix_port_id_action = pre_rix;
11163                         (*num_of_dest)++;
11164                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
11165                         break;
11166                 }
11167                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
11168                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
11169                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
11170                         /* Save the encap resource before sample */
11171                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
11172                         pre_r = dev_flow->dv.encap_decap;
11173                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
11174                                                            dev_flow,
11175                                                            attr->transfer,
11176                                                            error))
11177                                 return -rte_errno;
11178                         sample_act->dr_encap_action =
11179                                 dev_flow->dv.encap_decap->action;
11180                         sample_idx->rix_encap_decap =
11181                                 dev_flow->handle->dvh.rix_encap_decap;
11182                         sample_actions[sample_act->actions_num++] =
11183                                                 sample_act->dr_encap_action;
11184                         /* Recover the encap resource after sample */
11185                         dev_flow->dv.encap_decap = pre_r;
11186                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
11187                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
11188                         break;
11189                 default:
11190                         return rte_flow_error_set(error, EINVAL,
11191                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11192                                 NULL,
11193                                 "Not support for sampler action");
11194                 }
11195         }
11196         sample_act->action_flags = action_flags;
11197         res->ft_id = dev_flow->dv.group;
11198         if (attr->transfer) {
11199                 union {
11200                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
11201                         uint64_t set_action;
11202                 } action_ctx = { .set_action = 0 };
11203
11204                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
11205                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
11206                          MLX5_MODIFICATION_TYPE_SET);
11207                 MLX5_SET(set_action_in, action_ctx.action_in, field,
11208                          MLX5_MODI_META_REG_C_0);
11209                 MLX5_SET(set_action_in, action_ctx.action_in, data,
11210                          priv->vport_meta_tag);
11211                 res->set_action = action_ctx.set_action;
11212         } else if (attr->ingress) {
11213                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
11214         } else {
11215                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
11216         }
11217         return 0;
11218 }
11219
11220 /**
11221  * Convert Sample action to DV specification.
11222  *
11223  * @param[in] dev
11224  *   Pointer to rte_eth_dev structure.
11225  * @param[in, out] dev_flow
11226  *   Pointer to the mlx5_flow.
11227  * @param[in] num_of_dest
11228  *   The num of destination.
11229  * @param[in, out] res
11230  *   Pointer to sample resource.
11231  * @param[in, out] mdest_res
11232  *   Pointer to destination array resource.
11233  * @param[in] sample_actions
11234  *   Pointer to sample path actions list.
11235  * @param[in] action_flags
11236  *   Holds the actions detected until now.
11237  * @param[out] error
11238  *   Pointer to the error structure.
11239  *
11240  * @return
11241  *   0 on success, a negative errno value otherwise and rte_errno is set.
11242  */
11243 static int
11244 flow_dv_create_action_sample(struct rte_eth_dev *dev,
11245                              struct mlx5_flow *dev_flow,
11246                              uint32_t num_of_dest,
11247                              struct mlx5_flow_dv_sample_resource *res,
11248                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
11249                              void **sample_actions,
11250                              uint64_t action_flags,
11251                              struct rte_flow_error *error)
11252 {
11253         /* update normal path action resource into last index of array */
11254         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
11255         struct mlx5_flow_sub_actions_list *sample_act =
11256                                         &mdest_res->sample_act[dest_index];
11257         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11258         struct mlx5_flow_rss_desc *rss_desc;
11259         uint32_t normal_idx = 0;
11260         struct mlx5_hrxq *hrxq;
11261         uint32_t hrxq_idx;
11262
11263         MLX5_ASSERT(wks);
11264         rss_desc = &wks->rss_desc;
11265         if (num_of_dest > 1) {
11266                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
11267                         /* Handle QP action for mirroring */
11268                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11269                                                     rss_desc, &hrxq_idx);
11270                         if (!hrxq)
11271                                 return rte_flow_error_set
11272                                      (error, rte_errno,
11273                                       RTE_FLOW_ERROR_TYPE_ACTION,
11274                                       NULL,
11275                                       "cannot create rx queue");
11276                         normal_idx++;
11277                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
11278                         sample_act->dr_queue_action = hrxq->action;
11279                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11280                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11281                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
11282                 }
11283                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
11284                         normal_idx++;
11285                         mdest_res->sample_idx[dest_index].rix_encap_decap =
11286                                 dev_flow->handle->dvh.rix_encap_decap;
11287                         sample_act->dr_encap_action =
11288                                 dev_flow->dv.encap_decap->action;
11289                         dev_flow->handle->dvh.rix_encap_decap = 0;
11290                 }
11291                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
11292                         normal_idx++;
11293                         mdest_res->sample_idx[dest_index].rix_port_id_action =
11294                                 dev_flow->handle->rix_port_id_action;
11295                         sample_act->dr_port_id_action =
11296                                 dev_flow->dv.port_id_action->action;
11297                         dev_flow->handle->rix_port_id_action = 0;
11298                 }
11299                 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) {
11300                         normal_idx++;
11301                         mdest_res->sample_idx[dest_index].rix_jump =
11302                                 dev_flow->handle->rix_jump;
11303                         sample_act->dr_jump_action =
11304                                 dev_flow->dv.jump->action;
11305                         dev_flow->handle->rix_jump = 0;
11306                 }
11307                 sample_act->actions_num = normal_idx;
11308                 /* update sample action resource into first index of array */
11309                 mdest_res->ft_type = res->ft_type;
11310                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
11311                                 sizeof(struct mlx5_flow_sub_actions_idx));
11312                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
11313                                 sizeof(struct mlx5_flow_sub_actions_list));
11314                 mdest_res->num_of_dest = num_of_dest;
11315                 if (flow_dv_dest_array_resource_register(dev, mdest_res,
11316                                                          dev_flow, error))
11317                         return rte_flow_error_set(error, EINVAL,
11318                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11319                                                   NULL, "can't create sample "
11320                                                   "action");
11321         } else {
11322                 res->sub_actions = sample_actions;
11323                 if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
11324                         return rte_flow_error_set(error, EINVAL,
11325                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11326                                                   NULL,
11327                                                   "can't create sample action");
11328         }
11329         return 0;
11330 }
11331
11332 /**
11333  * Remove an ASO age action from age actions list.
11334  *
11335  * @param[in] dev
11336  *   Pointer to the Ethernet device structure.
11337  * @param[in] age
11338  *   Pointer to the aso age action handler.
11339  */
11340 static void
11341 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev,
11342                                 struct mlx5_aso_age_action *age)
11343 {
11344         struct mlx5_age_info *age_info;
11345         struct mlx5_age_param *age_param = &age->age_params;
11346         struct mlx5_priv *priv = dev->data->dev_private;
11347         uint16_t expected = AGE_CANDIDATE;
11348
11349         age_info = GET_PORT_AGE_INFO(priv);
11350         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
11351                                          AGE_FREE, false, __ATOMIC_RELAXED,
11352                                          __ATOMIC_RELAXED)) {
11353                 /**
11354                  * We need the lock even it is age timeout,
11355                  * since age action may still in process.
11356                  */
11357                 rte_spinlock_lock(&age_info->aged_sl);
11358                 LIST_REMOVE(age, next);
11359                 rte_spinlock_unlock(&age_info->aged_sl);
11360                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
11361         }
11362 }
11363
11364 /**
11365  * Release an ASO age action.
11366  *
11367  * @param[in] dev
11368  *   Pointer to the Ethernet device structure.
11369  * @param[in] age_idx
11370  *   Index of ASO age action to release.
11371  * @param[in] flow
11372  *   True if the release operation is during flow destroy operation.
11373  *   False if the release operation is during action destroy operation.
11374  *
11375  * @return
11376  *   0 when age action was removed, otherwise the number of references.
11377  */
11378 static int
11379 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
11380 {
11381         struct mlx5_priv *priv = dev->data->dev_private;
11382         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11383         struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx);
11384         uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED);
11385
11386         if (!ret) {
11387                 flow_dv_aso_age_remove_from_age(dev, age);
11388                 rte_spinlock_lock(&mng->free_sl);
11389                 LIST_INSERT_HEAD(&mng->free, age, next);
11390                 rte_spinlock_unlock(&mng->free_sl);
11391         }
11392         return ret;
11393 }
11394
11395 /**
11396  * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
11397  *
11398  * @param[in] dev
11399  *   Pointer to the Ethernet device structure.
11400  *
11401  * @return
11402  *   0 on success, otherwise negative errno value and rte_errno is set.
11403  */
11404 static int
11405 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
11406 {
11407         struct mlx5_priv *priv = dev->data->dev_private;
11408         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11409         void *old_pools = mng->pools;
11410         uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
11411         uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
11412         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
11413
11414         if (!pools) {
11415                 rte_errno = ENOMEM;
11416                 return -ENOMEM;
11417         }
11418         if (old_pools) {
11419                 memcpy(pools, old_pools,
11420                        mng->n * sizeof(struct mlx5_flow_counter_pool *));
11421                 mlx5_free(old_pools);
11422         } else {
11423                 /* First ASO flow hit allocation - starting ASO data-path. */
11424                 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh);
11425
11426                 if (ret) {
11427                         mlx5_free(pools);
11428                         return ret;
11429                 }
11430         }
11431         mng->n = resize;
11432         mng->pools = pools;
11433         return 0;
11434 }
11435
11436 /**
11437  * Create and initialize a new ASO aging pool.
11438  *
11439  * @param[in] dev
11440  *   Pointer to the Ethernet device structure.
11441  * @param[out] age_free
11442  *   Where to put the pointer of a new age action.
11443  *
11444  * @return
11445  *   The age actions pool pointer and @p age_free is set on success,
11446  *   NULL otherwise and rte_errno is set.
11447  */
11448 static struct mlx5_aso_age_pool *
11449 flow_dv_age_pool_create(struct rte_eth_dev *dev,
11450                         struct mlx5_aso_age_action **age_free)
11451 {
11452         struct mlx5_priv *priv = dev->data->dev_private;
11453         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11454         struct mlx5_aso_age_pool *pool = NULL;
11455         struct mlx5_devx_obj *obj = NULL;
11456         uint32_t i;
11457
11458         obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->ctx,
11459                                                     priv->sh->pdn);
11460         if (!obj) {
11461                 rte_errno = ENODATA;
11462                 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX.");
11463                 return NULL;
11464         }
11465         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
11466         if (!pool) {
11467                 claim_zero(mlx5_devx_cmd_destroy(obj));
11468                 rte_errno = ENOMEM;
11469                 return NULL;
11470         }
11471         pool->flow_hit_aso_obj = obj;
11472         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
11473         rte_spinlock_lock(&mng->resize_sl);
11474         pool->index = mng->next;
11475         /* Resize pools array if there is no room for the new pool in it. */
11476         if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) {
11477                 claim_zero(mlx5_devx_cmd_destroy(obj));
11478                 mlx5_free(pool);
11479                 rte_spinlock_unlock(&mng->resize_sl);
11480                 return NULL;
11481         }
11482         mng->pools[pool->index] = pool;
11483         mng->next++;
11484         rte_spinlock_unlock(&mng->resize_sl);
11485         /* Assign the first action in the new pool, the rest go to free list. */
11486         *age_free = &pool->actions[0];
11487         for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) {
11488                 pool->actions[i].offset = i;
11489                 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next);
11490         }
11491         return pool;
11492 }
11493
11494 /**
11495  * Allocate a ASO aging bit.
11496  *
11497  * @param[in] dev
11498  *   Pointer to the Ethernet device structure.
11499  * @param[out] error
11500  *   Pointer to the error structure.
11501  *
11502  * @return
11503  *   Index to ASO age action on success, 0 otherwise and rte_errno is set.
11504  */
11505 static uint32_t
11506 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
11507 {
11508         struct mlx5_priv *priv = dev->data->dev_private;
11509         const struct mlx5_aso_age_pool *pool;
11510         struct mlx5_aso_age_action *age_free = NULL;
11511         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11512
11513         MLX5_ASSERT(mng);
11514         /* Try to get the next free age action bit. */
11515         rte_spinlock_lock(&mng->free_sl);
11516         age_free = LIST_FIRST(&mng->free);
11517         if (age_free) {
11518                 LIST_REMOVE(age_free, next);
11519         } else if (!flow_dv_age_pool_create(dev, &age_free)) {
11520                 rte_spinlock_unlock(&mng->free_sl);
11521                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
11522                                    NULL, "failed to create ASO age pool");
11523                 return 0; /* 0 is an error. */
11524         }
11525         rte_spinlock_unlock(&mng->free_sl);
11526         pool = container_of
11527           ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL])
11528                   (age_free - age_free->offset), const struct mlx5_aso_age_pool,
11529                                                                        actions);
11530         if (!age_free->dr_action) {
11531                 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0,
11532                                                  error);
11533
11534                 if (reg_c < 0) {
11535                         rte_flow_error_set(error, rte_errno,
11536                                            RTE_FLOW_ERROR_TYPE_ACTION,
11537                                            NULL, "failed to get reg_c "
11538                                            "for ASO flow hit");
11539                         return 0; /* 0 is an error. */
11540                 }
11541 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
11542                 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso
11543                                 (priv->sh->rx_domain,
11544                                  pool->flow_hit_aso_obj->obj, age_free->offset,
11545                                  MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET,
11546                                  (reg_c - REG_C_0));
11547 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
11548                 if (!age_free->dr_action) {
11549                         rte_errno = errno;
11550                         rte_spinlock_lock(&mng->free_sl);
11551                         LIST_INSERT_HEAD(&mng->free, age_free, next);
11552                         rte_spinlock_unlock(&mng->free_sl);
11553                         rte_flow_error_set(error, rte_errno,
11554                                            RTE_FLOW_ERROR_TYPE_ACTION,
11555                                            NULL, "failed to create ASO "
11556                                            "flow hit action");
11557                         return 0; /* 0 is an error. */
11558                 }
11559         }
11560         __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED);
11561         return pool->index | ((age_free->offset + 1) << 16);
11562 }
11563
11564 /**
11565  * Initialize flow ASO age parameters.
11566  *
11567  * @param[in] dev
11568  *   Pointer to rte_eth_dev structure.
11569  * @param[in] age_idx
11570  *   Index of ASO age action.
11571  * @param[in] context
11572  *   Pointer to flow counter age context.
11573  * @param[in] timeout
11574  *   Aging timeout in seconds.
11575  *
11576  */
11577 static void
11578 flow_dv_aso_age_params_init(struct rte_eth_dev *dev,
11579                             uint32_t age_idx,
11580                             void *context,
11581                             uint32_t timeout)
11582 {
11583         struct mlx5_aso_age_action *aso_age;
11584
11585         aso_age = flow_aso_age_get_by_idx(dev, age_idx);
11586         MLX5_ASSERT(aso_age);
11587         aso_age->age_params.context = context;
11588         aso_age->age_params.timeout = timeout;
11589         aso_age->age_params.port_id = dev->data->port_id;
11590         __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0,
11591                          __ATOMIC_RELAXED);
11592         __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE,
11593                          __ATOMIC_RELAXED);
11594 }
11595
11596 static void
11597 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask,
11598                                const struct rte_flow_item_integrity *value,
11599                                void *headers_m, void *headers_v)
11600 {
11601         if (mask->l4_ok) {
11602                 /* application l4_ok filter aggregates all hardware l4 filters
11603                  * therefore hw l4_checksum_ok must be implicitly added here.
11604                  */
11605                 struct rte_flow_item_integrity local_item;
11606
11607                 local_item.l4_csum_ok = 1;
11608                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
11609                          local_item.l4_csum_ok);
11610                 if (value->l4_ok) {
11611                         /* application l4_ok = 1 matches sets both hw flags
11612                          * l4_ok and l4_checksum_ok flags to 1.
11613                          */
11614                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11615                                  l4_checksum_ok, local_item.l4_csum_ok);
11616                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok,
11617                                  mask->l4_ok);
11618                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok,
11619                                  value->l4_ok);
11620                 } else {
11621                         /* application l4_ok = 0 matches on hw flag
11622                          * l4_checksum_ok = 0 only.
11623                          */
11624                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11625                                  l4_checksum_ok, 0);
11626                 }
11627         } else if (mask->l4_csum_ok) {
11628                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
11629                          mask->l4_csum_ok);
11630                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
11631                          value->l4_csum_ok);
11632         }
11633 }
11634
11635 static void
11636 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask,
11637                                const struct rte_flow_item_integrity *value,
11638                                void *headers_m, void *headers_v,
11639                                bool is_ipv4)
11640 {
11641         if (mask->l3_ok) {
11642                 /* application l3_ok filter aggregates all hardware l3 filters
11643                  * therefore hw ipv4_checksum_ok must be implicitly added here.
11644                  */
11645                 struct rte_flow_item_integrity local_item;
11646
11647                 local_item.ipv4_csum_ok = !!is_ipv4;
11648                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
11649                          local_item.ipv4_csum_ok);
11650                 if (value->l3_ok) {
11651                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11652                                  ipv4_checksum_ok, local_item.ipv4_csum_ok);
11653                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok,
11654                                  mask->l3_ok);
11655                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok,
11656                                  value->l3_ok);
11657                 } else {
11658                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11659                                  ipv4_checksum_ok, 0);
11660                 }
11661         } else if (mask->ipv4_csum_ok) {
11662                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
11663                          mask->ipv4_csum_ok);
11664                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
11665                          value->ipv4_csum_ok);
11666         }
11667 }
11668
11669 static void
11670 flow_dv_translate_item_integrity(void *matcher, void *key,
11671                                  const struct rte_flow_item *head_item,
11672                                  const struct rte_flow_item *integrity_item)
11673 {
11674         const struct rte_flow_item_integrity *mask = integrity_item->mask;
11675         const struct rte_flow_item_integrity *value = integrity_item->spec;
11676         const struct rte_flow_item *tunnel_item, *end_item, *item;
11677         void *headers_m;
11678         void *headers_v;
11679         uint32_t l3_protocol;
11680
11681         if (!value)
11682                 return;
11683         if (!mask)
11684                 mask = &rte_flow_item_integrity_mask;
11685         if (value->level > 1) {
11686                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
11687                                          inner_headers);
11688                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
11689         } else {
11690                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
11691                                          outer_headers);
11692                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
11693         }
11694         tunnel_item = mlx5_flow_find_tunnel_item(head_item);
11695         if (value->level > 1) {
11696                 /* tunnel item was verified during the item validation */
11697                 item = tunnel_item;
11698                 end_item = mlx5_find_end_item(tunnel_item);
11699         } else {
11700                 item = head_item;
11701                 end_item = tunnel_item ? tunnel_item :
11702                            mlx5_find_end_item(integrity_item);
11703         }
11704         l3_protocol = mask->l3_ok ?
11705                       mlx5_flow_locate_proto_l3(&item, end_item) : 0;
11706         flow_dv_translate_integrity_l3(mask, value, headers_m, headers_v,
11707                                        l3_protocol == RTE_ETHER_TYPE_IPV4);
11708         flow_dv_translate_integrity_l4(mask, value, headers_m, headers_v);
11709 }
11710
11711 /**
11712  * Prepares DV flow counter with aging configuration.
11713  * Gets it by index when exists, creates a new one when doesn't.
11714  *
11715  * @param[in] dev
11716  *   Pointer to rte_eth_dev structure.
11717  * @param[in] dev_flow
11718  *   Pointer to the mlx5_flow.
11719  * @param[in, out] flow
11720  *   Pointer to the sub flow.
11721  * @param[in] count
11722  *   Pointer to the counter action configuration.
11723  * @param[in] age
11724  *   Pointer to the aging action configuration.
11725  * @param[out] error
11726  *   Pointer to the error structure.
11727  *
11728  * @return
11729  *   Pointer to the counter, NULL otherwise.
11730  */
11731 static struct mlx5_flow_counter *
11732 flow_dv_prepare_counter(struct rte_eth_dev *dev,
11733                         struct mlx5_flow *dev_flow,
11734                         struct rte_flow *flow,
11735                         const struct rte_flow_action_count *count,
11736                         const struct rte_flow_action_age *age,
11737                         struct rte_flow_error *error)
11738 {
11739         if (!flow->counter) {
11740                 flow->counter = flow_dv_translate_create_counter(dev, dev_flow,
11741                                                                  count, age);
11742                 if (!flow->counter) {
11743                         rte_flow_error_set(error, rte_errno,
11744                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11745                                            "cannot create counter object.");
11746                         return NULL;
11747                 }
11748         }
11749         return flow_dv_counter_get_by_idx(dev, flow->counter, NULL);
11750 }
11751
11752 /*
11753  * Release an ASO CT action by its own device.
11754  *
11755  * @param[in] dev
11756  *   Pointer to the Ethernet device structure.
11757  * @param[in] idx
11758  *   Index of ASO CT action to release.
11759  *
11760  * @return
11761  *   0 when CT action was removed, otherwise the number of references.
11762  */
11763 static inline int
11764 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx)
11765 {
11766         struct mlx5_priv *priv = dev->data->dev_private;
11767         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
11768         uint32_t ret;
11769         struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx);
11770         enum mlx5_aso_ct_state state =
11771                         __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
11772
11773         /* Cannot release when CT is in the ASO SQ. */
11774         if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY)
11775                 return -1;
11776         ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED);
11777         if (!ret) {
11778                 if (ct->dr_action_orig) {
11779 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
11780                         claim_zero(mlx5_glue->destroy_flow_action
11781                                         (ct->dr_action_orig));
11782 #endif
11783                         ct->dr_action_orig = NULL;
11784                 }
11785                 if (ct->dr_action_rply) {
11786 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
11787                         claim_zero(mlx5_glue->destroy_flow_action
11788                                         (ct->dr_action_rply));
11789 #endif
11790                         ct->dr_action_rply = NULL;
11791                 }
11792                 /* Clear the state to free, no need in 1st allocation. */
11793                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE);
11794                 rte_spinlock_lock(&mng->ct_sl);
11795                 LIST_INSERT_HEAD(&mng->free_cts, ct, next);
11796                 rte_spinlock_unlock(&mng->ct_sl);
11797         }
11798         return (int)ret;
11799 }
11800
11801 static inline int
11802 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx)
11803 {
11804         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx);
11805         uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx);
11806         struct rte_eth_dev *owndev = &rte_eth_devices[owner];
11807         RTE_SET_USED(dev);
11808
11809         MLX5_ASSERT(owner < RTE_MAX_ETHPORTS);
11810         if (dev->data->dev_started != 1)
11811                 return -1;
11812         return flow_dv_aso_ct_dev_release(owndev, idx);
11813 }
11814
11815 /*
11816  * Resize the ASO CT pools array by 64 pools.
11817  *
11818  * @param[in] dev
11819  *   Pointer to the Ethernet device structure.
11820  *
11821  * @return
11822  *   0 on success, otherwise negative errno value and rte_errno is set.
11823  */
11824 static int
11825 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev)
11826 {
11827         struct mlx5_priv *priv = dev->data->dev_private;
11828         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
11829         void *old_pools = mng->pools;
11830         /* Magic number now, need a macro. */
11831         uint32_t resize = mng->n + 64;
11832         uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize;
11833         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
11834
11835         if (!pools) {
11836                 rte_errno = ENOMEM;
11837                 return -rte_errno;
11838         }
11839         rte_rwlock_write_lock(&mng->resize_rwl);
11840         /* ASO SQ/QP was already initialized in the startup. */
11841         if (old_pools) {
11842                 /* Realloc could be an alternative choice. */
11843                 rte_memcpy(pools, old_pools,
11844                            mng->n * sizeof(struct mlx5_aso_ct_pool *));
11845                 mlx5_free(old_pools);
11846         }
11847         mng->n = resize;
11848         mng->pools = pools;
11849         rte_rwlock_write_unlock(&mng->resize_rwl);
11850         return 0;
11851 }
11852
11853 /*
11854  * Create and initialize a new ASO CT pool.
11855  *
11856  * @param[in] dev
11857  *   Pointer to the Ethernet device structure.
11858  * @param[out] ct_free
11859  *   Where to put the pointer of a new CT action.
11860  *
11861  * @return
11862  *   The CT actions pool pointer and @p ct_free is set on success,
11863  *   NULL otherwise and rte_errno is set.
11864  */
11865 static struct mlx5_aso_ct_pool *
11866 flow_dv_ct_pool_create(struct rte_eth_dev *dev,
11867                        struct mlx5_aso_ct_action **ct_free)
11868 {
11869         struct mlx5_priv *priv = dev->data->dev_private;
11870         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
11871         struct mlx5_aso_ct_pool *pool = NULL;
11872         struct mlx5_devx_obj *obj = NULL;
11873         uint32_t i;
11874         uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL);
11875
11876         obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->ctx,
11877                                                 priv->sh->pdn, log_obj_size);
11878         if (!obj) {
11879                 rte_errno = ENODATA;
11880                 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
11881                 return NULL;
11882         }
11883         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
11884         if (!pool) {
11885                 rte_errno = ENOMEM;
11886                 claim_zero(mlx5_devx_cmd_destroy(obj));
11887                 return NULL;
11888         }
11889         pool->devx_obj = obj;
11890         pool->index = mng->next;
11891         /* Resize pools array if there is no room for the new pool in it. */
11892         if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) {
11893                 claim_zero(mlx5_devx_cmd_destroy(obj));
11894                 mlx5_free(pool);
11895                 return NULL;
11896         }
11897         mng->pools[pool->index] = pool;
11898         mng->next++;
11899         /* Assign the first action in the new pool, the rest go to free list. */
11900         *ct_free = &pool->actions[0];
11901         /* Lock outside, the list operation is safe here. */
11902         for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) {
11903                 /* refcnt is 0 when allocating the memory. */
11904                 pool->actions[i].offset = i;
11905                 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next);
11906         }
11907         return pool;
11908 }
11909
11910 /*
11911  * Allocate a ASO CT action from free list.
11912  *
11913  * @param[in] dev
11914  *   Pointer to the Ethernet device structure.
11915  * @param[out] error
11916  *   Pointer to the error structure.
11917  *
11918  * @return
11919  *   Index to ASO CT action on success, 0 otherwise and rte_errno is set.
11920  */
11921 static uint32_t
11922 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
11923 {
11924         struct mlx5_priv *priv = dev->data->dev_private;
11925         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
11926         struct mlx5_aso_ct_action *ct = NULL;
11927         struct mlx5_aso_ct_pool *pool;
11928         uint8_t reg_c;
11929         uint32_t ct_idx;
11930
11931         MLX5_ASSERT(mng);
11932         if (!priv->config.devx) {
11933                 rte_errno = ENOTSUP;
11934                 return 0;
11935         }
11936         /* Get a free CT action, if no, a new pool will be created. */
11937         rte_spinlock_lock(&mng->ct_sl);
11938         ct = LIST_FIRST(&mng->free_cts);
11939         if (ct) {
11940                 LIST_REMOVE(ct, next);
11941         } else if (!flow_dv_ct_pool_create(dev, &ct)) {
11942                 rte_spinlock_unlock(&mng->ct_sl);
11943                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
11944                                    NULL, "failed to create ASO CT pool");
11945                 return 0;
11946         }
11947         rte_spinlock_unlock(&mng->ct_sl);
11948         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
11949         ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset);
11950         /* 0: inactive, 1: created, 2+: used by flows. */
11951         __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED);
11952         reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error);
11953         if (!ct->dr_action_orig) {
11954 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
11955                 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso
11956                         (priv->sh->rx_domain, pool->devx_obj->obj,
11957                          ct->offset,
11958                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR,
11959                          reg_c - REG_C_0);
11960 #else
11961                 RTE_SET_USED(reg_c);
11962 #endif
11963                 if (!ct->dr_action_orig) {
11964                         flow_dv_aso_ct_dev_release(dev, ct_idx);
11965                         rte_flow_error_set(error, rte_errno,
11966                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11967                                            "failed to create ASO CT action");
11968                         return 0;
11969                 }
11970         }
11971         if (!ct->dr_action_rply) {
11972 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
11973                 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso
11974                         (priv->sh->rx_domain, pool->devx_obj->obj,
11975                          ct->offset,
11976                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER,
11977                          reg_c - REG_C_0);
11978 #endif
11979                 if (!ct->dr_action_rply) {
11980                         flow_dv_aso_ct_dev_release(dev, ct_idx);
11981                         rte_flow_error_set(error, rte_errno,
11982                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11983                                            "failed to create ASO CT action");
11984                         return 0;
11985                 }
11986         }
11987         return ct_idx;
11988 }
11989
11990 /*
11991  * Create a conntrack object with context and actions by using ASO mechanism.
11992  *
11993  * @param[in] dev
11994  *   Pointer to rte_eth_dev structure.
11995  * @param[in] pro
11996  *   Pointer to conntrack information profile.
11997  * @param[out] error
11998  *   Pointer to the error structure.
11999  *
12000  * @return
12001  *   Index to conntrack object on success, 0 otherwise.
12002  */
12003 static uint32_t
12004 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev,
12005                                    const struct rte_flow_action_conntrack *pro,
12006                                    struct rte_flow_error *error)
12007 {
12008         struct mlx5_priv *priv = dev->data->dev_private;
12009         struct mlx5_dev_ctx_shared *sh = priv->sh;
12010         struct mlx5_aso_ct_action *ct;
12011         uint32_t idx;
12012
12013         if (!sh->ct_aso_en)
12014                 return rte_flow_error_set(error, ENOTSUP,
12015                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12016                                           "Connection is not supported");
12017         idx = flow_dv_aso_ct_alloc(dev, error);
12018         if (!idx)
12019                 return rte_flow_error_set(error, rte_errno,
12020                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12021                                           "Failed to allocate CT object");
12022         ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12023         if (mlx5_aso_ct_update_by_wqe(sh, ct, pro))
12024                 return rte_flow_error_set(error, EBUSY,
12025                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12026                                           "Failed to update CT");
12027         ct->is_original = !!pro->is_original_dir;
12028         ct->peer = pro->peer_port;
12029         return idx;
12030 }
12031
12032 /**
12033  * Fill the flow with DV spec, lock free
12034  * (mutex should be acquired by caller).
12035  *
12036  * @param[in] dev
12037  *   Pointer to rte_eth_dev structure.
12038  * @param[in, out] dev_flow
12039  *   Pointer to the sub flow.
12040  * @param[in] attr
12041  *   Pointer to the flow attributes.
12042  * @param[in] items
12043  *   Pointer to the list of items.
12044  * @param[in] actions
12045  *   Pointer to the list of actions.
12046  * @param[out] error
12047  *   Pointer to the error structure.
12048  *
12049  * @return
12050  *   0 on success, a negative errno value otherwise and rte_errno is set.
12051  */
12052 static int
12053 flow_dv_translate(struct rte_eth_dev *dev,
12054                   struct mlx5_flow *dev_flow,
12055                   const struct rte_flow_attr *attr,
12056                   const struct rte_flow_item items[],
12057                   const struct rte_flow_action actions[],
12058                   struct rte_flow_error *error)
12059 {
12060         struct mlx5_priv *priv = dev->data->dev_private;
12061         struct mlx5_dev_config *dev_conf = &priv->config;
12062         struct rte_flow *flow = dev_flow->flow;
12063         struct mlx5_flow_handle *handle = dev_flow->handle;
12064         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12065         struct mlx5_flow_rss_desc *rss_desc;
12066         uint64_t item_flags = 0;
12067         uint64_t last_item = 0;
12068         uint64_t action_flags = 0;
12069         struct mlx5_flow_dv_matcher matcher = {
12070                 .mask = {
12071                         .size = sizeof(matcher.mask.buf) -
12072                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
12073                 },
12074         };
12075         int actions_n = 0;
12076         bool actions_end = false;
12077         union {
12078                 struct mlx5_flow_dv_modify_hdr_resource res;
12079                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
12080                             sizeof(struct mlx5_modification_cmd) *
12081                             (MLX5_MAX_MODIFY_NUM + 1)];
12082         } mhdr_dummy;
12083         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
12084         const struct rte_flow_action_count *count = NULL;
12085         const struct rte_flow_action_age *non_shared_age = NULL;
12086         union flow_dv_attr flow_attr = { .attr = 0 };
12087         uint32_t tag_be;
12088         union mlx5_flow_tbl_key tbl_key;
12089         uint32_t modify_action_position = UINT32_MAX;
12090         void *match_mask = matcher.mask.buf;
12091         void *match_value = dev_flow->dv.value.buf;
12092         uint8_t next_protocol = 0xff;
12093         struct rte_vlan_hdr vlan = { 0 };
12094         struct mlx5_flow_dv_dest_array_resource mdest_res;
12095         struct mlx5_flow_dv_sample_resource sample_res;
12096         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
12097         const struct rte_flow_action_sample *sample = NULL;
12098         struct mlx5_flow_sub_actions_list *sample_act;
12099         uint32_t sample_act_pos = UINT32_MAX;
12100         uint32_t age_act_pos = UINT32_MAX;
12101         uint32_t num_of_dest = 0;
12102         int tmp_actions_n = 0;
12103         uint32_t table;
12104         int ret = 0;
12105         const struct mlx5_flow_tunnel *tunnel = NULL;
12106         struct flow_grp_info grp_info = {
12107                 .external = !!dev_flow->external,
12108                 .transfer = !!attr->transfer,
12109                 .fdb_def_rule = !!priv->fdb_def_rule,
12110                 .skip_scale = dev_flow->skip_scale &
12111                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
12112                 .std_tbl_fix = true,
12113         };
12114         const struct rte_flow_item *head_item = items;
12115
12116         if (!wks)
12117                 return rte_flow_error_set(error, ENOMEM,
12118                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12119                                           NULL,
12120                                           "failed to push flow workspace");
12121         rss_desc = &wks->rss_desc;
12122         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
12123         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
12124         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12125                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12126         /* update normal path action resource into last index of array */
12127         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
12128         if (is_tunnel_offload_active(dev)) {
12129                 if (dev_flow->tunnel) {
12130                         RTE_VERIFY(dev_flow->tof_type ==
12131                                    MLX5_TUNNEL_OFFLOAD_MISS_RULE);
12132                         tunnel = dev_flow->tunnel;
12133                 } else {
12134                         tunnel = mlx5_get_tof(items, actions,
12135                                               &dev_flow->tof_type);
12136                         dev_flow->tunnel = tunnel;
12137                 }
12138                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
12139                                         (dev, attr, tunnel, dev_flow->tof_type);
12140         }
12141         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12142                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12143         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
12144                                        &grp_info, error);
12145         if (ret)
12146                 return ret;
12147         dev_flow->dv.group = table;
12148         if (attr->transfer)
12149                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12150         /* number of actions must be set to 0 in case of dirty stack. */
12151         mhdr_res->actions_num = 0;
12152         if (is_flow_tunnel_match_rule(dev_flow->tof_type)) {
12153                 /*
12154                  * do not add decap action if match rule drops packet
12155                  * HW rejects rules with decap & drop
12156                  *
12157                  * if tunnel match rule was inserted before matching tunnel set
12158                  * rule flow table used in the match rule must be registered.
12159                  * current implementation handles that in the
12160                  * flow_dv_match_register() at the function end.
12161                  */
12162                 bool add_decap = true;
12163                 const struct rte_flow_action *ptr = actions;
12164
12165                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
12166                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
12167                                 add_decap = false;
12168                                 break;
12169                         }
12170                 }
12171                 if (add_decap) {
12172                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12173                                                            attr->transfer,
12174                                                            error))
12175                                 return -rte_errno;
12176                         dev_flow->dv.actions[actions_n++] =
12177                                         dev_flow->dv.encap_decap->action;
12178                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12179                 }
12180         }
12181         for (; !actions_end ; actions++) {
12182                 const struct rte_flow_action_queue *queue;
12183                 const struct rte_flow_action_rss *rss;
12184                 const struct rte_flow_action *action = actions;
12185                 const uint8_t *rss_key;
12186                 struct mlx5_flow_tbl_resource *tbl;
12187                 struct mlx5_aso_age_action *age_act;
12188                 struct mlx5_flow_counter *cnt_act;
12189                 uint32_t port_id = 0;
12190                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
12191                 int action_type = actions->type;
12192                 const struct rte_flow_action *found_action = NULL;
12193                 uint32_t jump_group = 0;
12194                 uint32_t owner_idx;
12195                 struct mlx5_aso_ct_action *ct;
12196
12197                 if (!mlx5_flow_os_action_supported(action_type))
12198                         return rte_flow_error_set(error, ENOTSUP,
12199                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12200                                                   actions,
12201                                                   "action not supported");
12202                 switch (action_type) {
12203                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
12204                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
12205                         break;
12206                 case RTE_FLOW_ACTION_TYPE_VOID:
12207                         break;
12208                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
12209                         if (flow_dv_translate_action_port_id(dev, action,
12210                                                              &port_id, error))
12211                                 return -rte_errno;
12212                         port_id_resource.port_id = port_id;
12213                         MLX5_ASSERT(!handle->rix_port_id_action);
12214                         if (flow_dv_port_id_action_resource_register
12215                             (dev, &port_id_resource, dev_flow, error))
12216                                 return -rte_errno;
12217                         dev_flow->dv.actions[actions_n++] =
12218                                         dev_flow->dv.port_id_action->action;
12219                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12220                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
12221                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12222                         num_of_dest++;
12223                         break;
12224                 case RTE_FLOW_ACTION_TYPE_FLAG:
12225                         action_flags |= MLX5_FLOW_ACTION_FLAG;
12226                         dev_flow->handle->mark = 1;
12227                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12228                                 struct rte_flow_action_mark mark = {
12229                                         .id = MLX5_FLOW_MARK_DEFAULT,
12230                                 };
12231
12232                                 if (flow_dv_convert_action_mark(dev, &mark,
12233                                                                 mhdr_res,
12234                                                                 error))
12235                                         return -rte_errno;
12236                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12237                                 break;
12238                         }
12239                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
12240                         /*
12241                          * Only one FLAG or MARK is supported per device flow
12242                          * right now. So the pointer to the tag resource must be
12243                          * zero before the register process.
12244                          */
12245                         MLX5_ASSERT(!handle->dvh.rix_tag);
12246                         if (flow_dv_tag_resource_register(dev, tag_be,
12247                                                           dev_flow, error))
12248                                 return -rte_errno;
12249                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12250                         dev_flow->dv.actions[actions_n++] =
12251                                         dev_flow->dv.tag_resource->action;
12252                         break;
12253                 case RTE_FLOW_ACTION_TYPE_MARK:
12254                         action_flags |= MLX5_FLOW_ACTION_MARK;
12255                         dev_flow->handle->mark = 1;
12256                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12257                                 const struct rte_flow_action_mark *mark =
12258                                         (const struct rte_flow_action_mark *)
12259                                                 actions->conf;
12260
12261                                 if (flow_dv_convert_action_mark(dev, mark,
12262                                                                 mhdr_res,
12263                                                                 error))
12264                                         return -rte_errno;
12265                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12266                                 break;
12267                         }
12268                         /* Fall-through */
12269                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
12270                         /* Legacy (non-extensive) MARK action. */
12271                         tag_be = mlx5_flow_mark_set
12272                               (((const struct rte_flow_action_mark *)
12273                                (actions->conf))->id);
12274                         MLX5_ASSERT(!handle->dvh.rix_tag);
12275                         if (flow_dv_tag_resource_register(dev, tag_be,
12276                                                           dev_flow, error))
12277                                 return -rte_errno;
12278                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12279                         dev_flow->dv.actions[actions_n++] =
12280                                         dev_flow->dv.tag_resource->action;
12281                         break;
12282                 case RTE_FLOW_ACTION_TYPE_SET_META:
12283                         if (flow_dv_convert_action_set_meta
12284                                 (dev, mhdr_res, attr,
12285                                  (const struct rte_flow_action_set_meta *)
12286                                   actions->conf, error))
12287                                 return -rte_errno;
12288                         action_flags |= MLX5_FLOW_ACTION_SET_META;
12289                         break;
12290                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
12291                         if (flow_dv_convert_action_set_tag
12292                                 (dev, mhdr_res,
12293                                  (const struct rte_flow_action_set_tag *)
12294                                   actions->conf, error))
12295                                 return -rte_errno;
12296                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12297                         break;
12298                 case RTE_FLOW_ACTION_TYPE_DROP:
12299                         action_flags |= MLX5_FLOW_ACTION_DROP;
12300                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
12301                         break;
12302                 case RTE_FLOW_ACTION_TYPE_QUEUE:
12303                         queue = actions->conf;
12304                         rss_desc->queue_num = 1;
12305                         rss_desc->queue[0] = queue->index;
12306                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
12307                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
12308                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
12309                         num_of_dest++;
12310                         break;
12311                 case RTE_FLOW_ACTION_TYPE_RSS:
12312                         rss = actions->conf;
12313                         memcpy(rss_desc->queue, rss->queue,
12314                                rss->queue_num * sizeof(uint16_t));
12315                         rss_desc->queue_num = rss->queue_num;
12316                         /* NULL RSS key indicates default RSS key. */
12317                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
12318                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
12319                         /*
12320                          * rss->level and rss.types should be set in advance
12321                          * when expanding items for RSS.
12322                          */
12323                         action_flags |= MLX5_FLOW_ACTION_RSS;
12324                         dev_flow->handle->fate_action = rss_desc->shared_rss ?
12325                                 MLX5_FLOW_FATE_SHARED_RSS :
12326                                 MLX5_FLOW_FATE_QUEUE;
12327                         break;
12328                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
12329                         flow->age = (uint32_t)(uintptr_t)(action->conf);
12330                         age_act = flow_aso_age_get_by_idx(dev, flow->age);
12331                         __atomic_fetch_add(&age_act->refcnt, 1,
12332                                            __ATOMIC_RELAXED);
12333                         age_act_pos = actions_n++;
12334                         action_flags |= MLX5_FLOW_ACTION_AGE;
12335                         break;
12336                 case RTE_FLOW_ACTION_TYPE_AGE:
12337                         non_shared_age = action->conf;
12338                         age_act_pos = actions_n++;
12339                         action_flags |= MLX5_FLOW_ACTION_AGE;
12340                         break;
12341                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
12342                         flow->counter = (uint32_t)(uintptr_t)(action->conf);
12343                         cnt_act = flow_dv_counter_get_by_idx(dev, flow->counter,
12344                                                              NULL);
12345                         __atomic_fetch_add(&cnt_act->shared_info.refcnt, 1,
12346                                            __ATOMIC_RELAXED);
12347                         /* Save information first, will apply later. */
12348                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12349                         break;
12350                 case RTE_FLOW_ACTION_TYPE_COUNT:
12351                         if (!dev_conf->devx) {
12352                                 return rte_flow_error_set
12353                                               (error, ENOTSUP,
12354                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12355                                                NULL,
12356                                                "count action not supported");
12357                         }
12358                         /* Save information first, will apply later. */
12359                         count = action->conf;
12360                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12361                         break;
12362                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
12363                         dev_flow->dv.actions[actions_n++] =
12364                                                 priv->sh->pop_vlan_action;
12365                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
12366                         break;
12367                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
12368                         if (!(action_flags &
12369                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
12370                                 flow_dev_get_vlan_info_from_items(items, &vlan);
12371                         vlan.eth_proto = rte_be_to_cpu_16
12372                              ((((const struct rte_flow_action_of_push_vlan *)
12373                                                    actions->conf)->ethertype));
12374                         found_action = mlx5_flow_find_action
12375                                         (actions + 1,
12376                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
12377                         if (found_action)
12378                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12379                         found_action = mlx5_flow_find_action
12380                                         (actions + 1,
12381                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
12382                         if (found_action)
12383                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12384                         if (flow_dv_create_action_push_vlan
12385                                             (dev, attr, &vlan, dev_flow, error))
12386                                 return -rte_errno;
12387                         dev_flow->dv.actions[actions_n++] =
12388                                         dev_flow->dv.push_vlan_res->action;
12389                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
12390                         break;
12391                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
12392                         /* of_vlan_push action handled this action */
12393                         MLX5_ASSERT(action_flags &
12394                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
12395                         break;
12396                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
12397                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
12398                                 break;
12399                         flow_dev_get_vlan_info_from_items(items, &vlan);
12400                         mlx5_update_vlan_vid_pcp(actions, &vlan);
12401                         /* If no VLAN push - this is a modify header action */
12402                         if (flow_dv_convert_action_modify_vlan_vid
12403                                                 (mhdr_res, actions, error))
12404                                 return -rte_errno;
12405                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
12406                         break;
12407                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
12408                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
12409                         if (flow_dv_create_action_l2_encap(dev, actions,
12410                                                            dev_flow,
12411                                                            attr->transfer,
12412                                                            error))
12413                                 return -rte_errno;
12414                         dev_flow->dv.actions[actions_n++] =
12415                                         dev_flow->dv.encap_decap->action;
12416                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12417                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12418                                 sample_act->action_flags |=
12419                                                         MLX5_FLOW_ACTION_ENCAP;
12420                         break;
12421                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
12422                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
12423                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12424                                                            attr->transfer,
12425                                                            error))
12426                                 return -rte_errno;
12427                         dev_flow->dv.actions[actions_n++] =
12428                                         dev_flow->dv.encap_decap->action;
12429                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12430                         break;
12431                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
12432                         /* Handle encap with preceding decap. */
12433                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
12434                                 if (flow_dv_create_action_raw_encap
12435                                         (dev, actions, dev_flow, attr, error))
12436                                         return -rte_errno;
12437                                 dev_flow->dv.actions[actions_n++] =
12438                                         dev_flow->dv.encap_decap->action;
12439                         } else {
12440                                 /* Handle encap without preceding decap. */
12441                                 if (flow_dv_create_action_l2_encap
12442                                     (dev, actions, dev_flow, attr->transfer,
12443                                      error))
12444                                         return -rte_errno;
12445                                 dev_flow->dv.actions[actions_n++] =
12446                                         dev_flow->dv.encap_decap->action;
12447                         }
12448                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12449                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12450                                 sample_act->action_flags |=
12451                                                         MLX5_FLOW_ACTION_ENCAP;
12452                         break;
12453                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
12454                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
12455                                 ;
12456                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
12457                                 if (flow_dv_create_action_l2_decap
12458                                     (dev, dev_flow, attr->transfer, error))
12459                                         return -rte_errno;
12460                                 dev_flow->dv.actions[actions_n++] =
12461                                         dev_flow->dv.encap_decap->action;
12462                         }
12463                         /* If decap is followed by encap, handle it at encap. */
12464                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12465                         break;
12466                 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP:
12467                         dev_flow->dv.actions[actions_n++] =
12468                                 (void *)(uintptr_t)action->conf;
12469                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12470                         break;
12471                 case RTE_FLOW_ACTION_TYPE_JUMP:
12472                         jump_group = ((const struct rte_flow_action_jump *)
12473                                                         action->conf)->group;
12474                         grp_info.std_tbl_fix = 0;
12475                         if (dev_flow->skip_scale &
12476                                 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT))
12477                                 grp_info.skip_scale = 1;
12478                         else
12479                                 grp_info.skip_scale = 0;
12480                         ret = mlx5_flow_group_to_table(dev, tunnel,
12481                                                        jump_group,
12482                                                        &table,
12483                                                        &grp_info, error);
12484                         if (ret)
12485                                 return ret;
12486                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
12487                                                        attr->transfer,
12488                                                        !!dev_flow->external,
12489                                                        tunnel, jump_group, 0,
12490                                                        0, error);
12491                         if (!tbl)
12492                                 return rte_flow_error_set
12493                                                 (error, errno,
12494                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12495                                                  NULL,
12496                                                  "cannot create jump action.");
12497                         if (flow_dv_jump_tbl_resource_register
12498                             (dev, tbl, dev_flow, error)) {
12499                                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
12500                                 return rte_flow_error_set
12501                                                 (error, errno,
12502                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12503                                                  NULL,
12504                                                  "cannot create jump action.");
12505                         }
12506                         dev_flow->dv.actions[actions_n++] =
12507                                         dev_flow->dv.jump->action;
12508                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12509                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
12510                         sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP;
12511                         num_of_dest++;
12512                         break;
12513                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
12514                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
12515                         if (flow_dv_convert_action_modify_mac
12516                                         (mhdr_res, actions, error))
12517                                 return -rte_errno;
12518                         action_flags |= actions->type ==
12519                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
12520                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
12521                                         MLX5_FLOW_ACTION_SET_MAC_DST;
12522                         break;
12523                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
12524                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
12525                         if (flow_dv_convert_action_modify_ipv4
12526                                         (mhdr_res, actions, error))
12527                                 return -rte_errno;
12528                         action_flags |= actions->type ==
12529                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
12530                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
12531                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
12532                         break;
12533                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
12534                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
12535                         if (flow_dv_convert_action_modify_ipv6
12536                                         (mhdr_res, actions, error))
12537                                 return -rte_errno;
12538                         action_flags |= actions->type ==
12539                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
12540                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
12541                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
12542                         break;
12543                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
12544                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
12545                         if (flow_dv_convert_action_modify_tp
12546                                         (mhdr_res, actions, items,
12547                                          &flow_attr, dev_flow, !!(action_flags &
12548                                          MLX5_FLOW_ACTION_DECAP), error))
12549                                 return -rte_errno;
12550                         action_flags |= actions->type ==
12551                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
12552                                         MLX5_FLOW_ACTION_SET_TP_SRC :
12553                                         MLX5_FLOW_ACTION_SET_TP_DST;
12554                         break;
12555                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
12556                         if (flow_dv_convert_action_modify_dec_ttl
12557                                         (mhdr_res, items, &flow_attr, dev_flow,
12558                                          !!(action_flags &
12559                                          MLX5_FLOW_ACTION_DECAP), error))
12560                                 return -rte_errno;
12561                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
12562                         break;
12563                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
12564                         if (flow_dv_convert_action_modify_ttl
12565                                         (mhdr_res, actions, items, &flow_attr,
12566                                          dev_flow, !!(action_flags &
12567                                          MLX5_FLOW_ACTION_DECAP), error))
12568                                 return -rte_errno;
12569                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
12570                         break;
12571                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
12572                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
12573                         if (flow_dv_convert_action_modify_tcp_seq
12574                                         (mhdr_res, actions, error))
12575                                 return -rte_errno;
12576                         action_flags |= actions->type ==
12577                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
12578                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
12579                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
12580                         break;
12581
12582                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
12583                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
12584                         if (flow_dv_convert_action_modify_tcp_ack
12585                                         (mhdr_res, actions, error))
12586                                 return -rte_errno;
12587                         action_flags |= actions->type ==
12588                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
12589                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
12590                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
12591                         break;
12592                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
12593                         if (flow_dv_convert_action_set_reg
12594                                         (mhdr_res, actions, error))
12595                                 return -rte_errno;
12596                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12597                         break;
12598                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
12599                         if (flow_dv_convert_action_copy_mreg
12600                                         (dev, mhdr_res, actions, error))
12601                                 return -rte_errno;
12602                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12603                         break;
12604                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
12605                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
12606                         dev_flow->handle->fate_action =
12607                                         MLX5_FLOW_FATE_DEFAULT_MISS;
12608                         break;
12609                 case RTE_FLOW_ACTION_TYPE_METER:
12610                         if (!wks->fm)
12611                                 return rte_flow_error_set(error, rte_errno,
12612                                         RTE_FLOW_ERROR_TYPE_ACTION,
12613                                         NULL, "Failed to get meter in flow.");
12614                         /* Set the meter action. */
12615                         dev_flow->dv.actions[actions_n++] =
12616                                 wks->fm->meter_action;
12617                         action_flags |= MLX5_FLOW_ACTION_METER;
12618                         break;
12619                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
12620                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
12621                                                               actions, error))
12622                                 return -rte_errno;
12623                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
12624                         break;
12625                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
12626                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
12627                                                               actions, error))
12628                                 return -rte_errno;
12629                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
12630                         break;
12631                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
12632                         sample_act_pos = actions_n;
12633                         sample = (const struct rte_flow_action_sample *)
12634                                  action->conf;
12635                         actions_n++;
12636                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
12637                         /* put encap action into group if work with port id */
12638                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
12639                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
12640                                 sample_act->action_flags |=
12641                                                         MLX5_FLOW_ACTION_ENCAP;
12642                         break;
12643                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
12644                         if (flow_dv_convert_action_modify_field
12645                                         (dev, mhdr_res, actions, attr, error))
12646                                 return -rte_errno;
12647                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
12648                         break;
12649                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
12650                         owner_idx = (uint32_t)(uintptr_t)action->conf;
12651                         ct = flow_aso_ct_get_by_idx(dev, owner_idx);
12652                         if (!ct)
12653                                 return rte_flow_error_set(error, EINVAL,
12654                                                 RTE_FLOW_ERROR_TYPE_ACTION,
12655                                                 NULL,
12656                                                 "Failed to get CT object.");
12657                         if (mlx5_aso_ct_available(priv->sh, ct))
12658                                 return rte_flow_error_set(error, rte_errno,
12659                                                 RTE_FLOW_ERROR_TYPE_ACTION,
12660                                                 NULL,
12661                                                 "CT is unavailable.");
12662                         if (ct->is_original)
12663                                 dev_flow->dv.actions[actions_n] =
12664                                                         ct->dr_action_orig;
12665                         else
12666                                 dev_flow->dv.actions[actions_n] =
12667                                                         ct->dr_action_rply;
12668                         flow->indirect_type = MLX5_INDIRECT_ACTION_TYPE_CT;
12669                         flow->ct = owner_idx;
12670                         __atomic_fetch_add(&ct->refcnt, 1, __ATOMIC_RELAXED);
12671                         actions_n++;
12672                         action_flags |= MLX5_FLOW_ACTION_CT;
12673                         break;
12674                 case RTE_FLOW_ACTION_TYPE_END:
12675                         actions_end = true;
12676                         if (mhdr_res->actions_num) {
12677                                 /* create modify action if needed. */
12678                                 if (flow_dv_modify_hdr_resource_register
12679                                         (dev, mhdr_res, dev_flow, error))
12680                                         return -rte_errno;
12681                                 dev_flow->dv.actions[modify_action_position] =
12682                                         handle->dvh.modify_hdr->action;
12683                         }
12684                         /*
12685                          * Handle AGE and COUNT action by single HW counter
12686                          * when they are not shared.
12687                          */
12688                         if (action_flags & MLX5_FLOW_ACTION_AGE) {
12689                                 if ((non_shared_age &&
12690                                      count && !count->shared) ||
12691                                     !(priv->sh->flow_hit_aso_en &&
12692                                       (attr->group || attr->transfer))) {
12693                                         /* Creates age by counters. */
12694                                         cnt_act = flow_dv_prepare_counter
12695                                                                 (dev, dev_flow,
12696                                                                  flow, count,
12697                                                                  non_shared_age,
12698                                                                  error);
12699                                         if (!cnt_act)
12700                                                 return -rte_errno;
12701                                         dev_flow->dv.actions[age_act_pos] =
12702                                                                 cnt_act->action;
12703                                         break;
12704                                 }
12705                                 if (!flow->age && non_shared_age) {
12706                                         flow->age = flow_dv_aso_age_alloc
12707                                                                 (dev, error);
12708                                         if (!flow->age)
12709                                                 return -rte_errno;
12710                                         flow_dv_aso_age_params_init
12711                                                     (dev, flow->age,
12712                                                      non_shared_age->context ?
12713                                                      non_shared_age->context :
12714                                                      (void *)(uintptr_t)
12715                                                      (dev_flow->flow_idx),
12716                                                      non_shared_age->timeout);
12717                                 }
12718                                 age_act = flow_aso_age_get_by_idx(dev,
12719                                                                   flow->age);
12720                                 dev_flow->dv.actions[age_act_pos] =
12721                                                              age_act->dr_action;
12722                         }
12723                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
12724                                 /*
12725                                  * Create one count action, to be used
12726                                  * by all sub-flows.
12727                                  */
12728                                 cnt_act = flow_dv_prepare_counter(dev, dev_flow,
12729                                                                   flow, count,
12730                                                                   NULL, error);
12731                                 if (!cnt_act)
12732                                         return -rte_errno;
12733                                 dev_flow->dv.actions[actions_n++] =
12734                                                                 cnt_act->action;
12735                         }
12736                 default:
12737                         break;
12738                 }
12739                 if (mhdr_res->actions_num &&
12740                     modify_action_position == UINT32_MAX)
12741                         modify_action_position = actions_n++;
12742         }
12743         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
12744                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
12745                 int item_type = items->type;
12746
12747                 if (!mlx5_flow_os_item_supported(item_type))
12748                         return rte_flow_error_set(error, ENOTSUP,
12749                                                   RTE_FLOW_ERROR_TYPE_ITEM,
12750                                                   NULL, "item not supported");
12751                 switch (item_type) {
12752                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
12753                         flow_dv_translate_item_port_id
12754                                 (dev, match_mask, match_value, items, attr);
12755                         last_item = MLX5_FLOW_ITEM_PORT_ID;
12756                         break;
12757                 case RTE_FLOW_ITEM_TYPE_ETH:
12758                         flow_dv_translate_item_eth(match_mask, match_value,
12759                                                    items, tunnel,
12760                                                    dev_flow->dv.group);
12761                         matcher.priority = action_flags &
12762                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
12763                                         !dev_flow->external ?
12764                                         MLX5_PRIORITY_MAP_L3 :
12765                                         MLX5_PRIORITY_MAP_L2;
12766                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
12767                                              MLX5_FLOW_LAYER_OUTER_L2;
12768                         break;
12769                 case RTE_FLOW_ITEM_TYPE_VLAN:
12770                         flow_dv_translate_item_vlan(dev_flow,
12771                                                     match_mask, match_value,
12772                                                     items, tunnel,
12773                                                     dev_flow->dv.group);
12774                         matcher.priority = MLX5_PRIORITY_MAP_L2;
12775                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
12776                                               MLX5_FLOW_LAYER_INNER_VLAN) :
12777                                              (MLX5_FLOW_LAYER_OUTER_L2 |
12778                                               MLX5_FLOW_LAYER_OUTER_VLAN);
12779                         break;
12780                 case RTE_FLOW_ITEM_TYPE_IPV4:
12781                         mlx5_flow_tunnel_ip_check(items, next_protocol,
12782                                                   &item_flags, &tunnel);
12783                         flow_dv_translate_item_ipv4(match_mask, match_value,
12784                                                     items, tunnel,
12785                                                     dev_flow->dv.group);
12786                         matcher.priority = MLX5_PRIORITY_MAP_L3;
12787                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
12788                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
12789                         if (items->mask != NULL &&
12790                             ((const struct rte_flow_item_ipv4 *)
12791                              items->mask)->hdr.next_proto_id) {
12792                                 next_protocol =
12793                                         ((const struct rte_flow_item_ipv4 *)
12794                                          (items->spec))->hdr.next_proto_id;
12795                                 next_protocol &=
12796                                         ((const struct rte_flow_item_ipv4 *)
12797                                          (items->mask))->hdr.next_proto_id;
12798                         } else {
12799                                 /* Reset for inner layer. */
12800                                 next_protocol = 0xff;
12801                         }
12802                         break;
12803                 case RTE_FLOW_ITEM_TYPE_IPV6:
12804                         mlx5_flow_tunnel_ip_check(items, next_protocol,
12805                                                   &item_flags, &tunnel);
12806                         flow_dv_translate_item_ipv6(match_mask, match_value,
12807                                                     items, tunnel,
12808                                                     dev_flow->dv.group);
12809                         matcher.priority = MLX5_PRIORITY_MAP_L3;
12810                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
12811                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
12812                         if (items->mask != NULL &&
12813                             ((const struct rte_flow_item_ipv6 *)
12814                              items->mask)->hdr.proto) {
12815                                 next_protocol =
12816                                         ((const struct rte_flow_item_ipv6 *)
12817                                          items->spec)->hdr.proto;
12818                                 next_protocol &=
12819                                         ((const struct rte_flow_item_ipv6 *)
12820                                          items->mask)->hdr.proto;
12821                         } else {
12822                                 /* Reset for inner layer. */
12823                                 next_protocol = 0xff;
12824                         }
12825                         break;
12826                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
12827                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
12828                                                              match_value,
12829                                                              items, tunnel);
12830                         last_item = tunnel ?
12831                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
12832                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
12833                         if (items->mask != NULL &&
12834                             ((const struct rte_flow_item_ipv6_frag_ext *)
12835                              items->mask)->hdr.next_header) {
12836                                 next_protocol =
12837                                 ((const struct rte_flow_item_ipv6_frag_ext *)
12838                                  items->spec)->hdr.next_header;
12839                                 next_protocol &=
12840                                 ((const struct rte_flow_item_ipv6_frag_ext *)
12841                                  items->mask)->hdr.next_header;
12842                         } else {
12843                                 /* Reset for inner layer. */
12844                                 next_protocol = 0xff;
12845                         }
12846                         break;
12847                 case RTE_FLOW_ITEM_TYPE_TCP:
12848                         flow_dv_translate_item_tcp(match_mask, match_value,
12849                                                    items, tunnel);
12850                         matcher.priority = MLX5_PRIORITY_MAP_L4;
12851                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
12852                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
12853                         break;
12854                 case RTE_FLOW_ITEM_TYPE_UDP:
12855                         flow_dv_translate_item_udp(match_mask, match_value,
12856                                                    items, tunnel);
12857                         matcher.priority = MLX5_PRIORITY_MAP_L4;
12858                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
12859                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
12860                         break;
12861                 case RTE_FLOW_ITEM_TYPE_GRE:
12862                         flow_dv_translate_item_gre(match_mask, match_value,
12863                                                    items, tunnel);
12864                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12865                         last_item = MLX5_FLOW_LAYER_GRE;
12866                         break;
12867                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
12868                         flow_dv_translate_item_gre_key(match_mask,
12869                                                        match_value, items);
12870                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
12871                         break;
12872                 case RTE_FLOW_ITEM_TYPE_NVGRE:
12873                         flow_dv_translate_item_nvgre(match_mask, match_value,
12874                                                      items, tunnel);
12875                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12876                         last_item = MLX5_FLOW_LAYER_GRE;
12877                         break;
12878                 case RTE_FLOW_ITEM_TYPE_VXLAN:
12879                         flow_dv_translate_item_vxlan(match_mask, match_value,
12880                                                      items, tunnel);
12881                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12882                         last_item = MLX5_FLOW_LAYER_VXLAN;
12883                         break;
12884                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
12885                         flow_dv_translate_item_vxlan_gpe(match_mask,
12886                                                          match_value, items,
12887                                                          tunnel);
12888                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12889                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
12890                         break;
12891                 case RTE_FLOW_ITEM_TYPE_GENEVE:
12892                         flow_dv_translate_item_geneve(match_mask, match_value,
12893                                                       items, tunnel);
12894                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12895                         last_item = MLX5_FLOW_LAYER_GENEVE;
12896                         break;
12897                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
12898                         ret = flow_dv_translate_item_geneve_opt(dev, match_mask,
12899                                                           match_value,
12900                                                           items, error);
12901                         if (ret)
12902                                 return rte_flow_error_set(error, -ret,
12903                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
12904                                         "cannot create GENEVE TLV option");
12905                         flow->geneve_tlv_option = 1;
12906                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
12907                         break;
12908                 case RTE_FLOW_ITEM_TYPE_MPLS:
12909                         flow_dv_translate_item_mpls(match_mask, match_value,
12910                                                     items, last_item, tunnel);
12911                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12912                         last_item = MLX5_FLOW_LAYER_MPLS;
12913                         break;
12914                 case RTE_FLOW_ITEM_TYPE_MARK:
12915                         flow_dv_translate_item_mark(dev, match_mask,
12916                                                     match_value, items);
12917                         last_item = MLX5_FLOW_ITEM_MARK;
12918                         break;
12919                 case RTE_FLOW_ITEM_TYPE_META:
12920                         flow_dv_translate_item_meta(dev, match_mask,
12921                                                     match_value, attr, items);
12922                         last_item = MLX5_FLOW_ITEM_METADATA;
12923                         break;
12924                 case RTE_FLOW_ITEM_TYPE_ICMP:
12925                         flow_dv_translate_item_icmp(match_mask, match_value,
12926                                                     items, tunnel);
12927                         last_item = MLX5_FLOW_LAYER_ICMP;
12928                         break;
12929                 case RTE_FLOW_ITEM_TYPE_ICMP6:
12930                         flow_dv_translate_item_icmp6(match_mask, match_value,
12931                                                       items, tunnel);
12932                         last_item = MLX5_FLOW_LAYER_ICMP6;
12933                         break;
12934                 case RTE_FLOW_ITEM_TYPE_TAG:
12935                         flow_dv_translate_item_tag(dev, match_mask,
12936                                                    match_value, items);
12937                         last_item = MLX5_FLOW_ITEM_TAG;
12938                         break;
12939                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
12940                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
12941                                                         match_value, items);
12942                         last_item = MLX5_FLOW_ITEM_TAG;
12943                         break;
12944                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
12945                         flow_dv_translate_item_tx_queue(dev, match_mask,
12946                                                         match_value,
12947                                                         items);
12948                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
12949                         break;
12950                 case RTE_FLOW_ITEM_TYPE_GTP:
12951                         flow_dv_translate_item_gtp(match_mask, match_value,
12952                                                    items, tunnel);
12953                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
12954                         last_item = MLX5_FLOW_LAYER_GTP;
12955                         break;
12956                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
12957                         ret = flow_dv_translate_item_gtp_psc(match_mask,
12958                                                           match_value,
12959                                                           items);
12960                         if (ret)
12961                                 return rte_flow_error_set(error, -ret,
12962                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
12963                                         "cannot create GTP PSC item");
12964                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
12965                         break;
12966                 case RTE_FLOW_ITEM_TYPE_ECPRI:
12967                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
12968                                 /* Create it only the first time to be used. */
12969                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
12970                                 if (ret)
12971                                         return rte_flow_error_set
12972                                                 (error, -ret,
12973                                                 RTE_FLOW_ERROR_TYPE_ITEM,
12974                                                 NULL,
12975                                                 "cannot create eCPRI parser");
12976                         }
12977                         /* Adjust the length matcher and device flow value. */
12978                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
12979                         dev_flow->dv.value.size =
12980                                         MLX5_ST_SZ_BYTES(fte_match_param);
12981                         flow_dv_translate_item_ecpri(dev, match_mask,
12982                                                      match_value, items);
12983                         /* No other protocol should follow eCPRI layer. */
12984                         last_item = MLX5_FLOW_LAYER_ECPRI;
12985                         break;
12986                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
12987                         flow_dv_translate_item_integrity(match_mask,
12988                                                          match_value,
12989                                                          head_item, items);
12990                         break;
12991                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
12992                         flow_dv_translate_item_aso_ct(dev, match_mask,
12993                                                       match_value, items);
12994                         break;
12995                 default:
12996                         break;
12997                 }
12998                 item_flags |= last_item;
12999         }
13000         /*
13001          * When E-Switch mode is enabled, we have two cases where we need to
13002          * set the source port manually.
13003          * The first one, is in case of Nic steering rule, and the second is
13004          * E-Switch rule where no port_id item was found. In both cases
13005          * the source port is set according the current port in use.
13006          */
13007         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
13008             (priv->representor || priv->master)) {
13009                 if (flow_dv_translate_item_port_id(dev, match_mask,
13010                                                    match_value, NULL, attr))
13011                         return -rte_errno;
13012         }
13013 #ifdef RTE_LIBRTE_MLX5_DEBUG
13014         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
13015                                               dev_flow->dv.value.buf));
13016 #endif
13017         /*
13018          * Layers may be already initialized from prefix flow if this dev_flow
13019          * is the suffix flow.
13020          */
13021         handle->layers |= item_flags;
13022         if (action_flags & MLX5_FLOW_ACTION_RSS)
13023                 flow_dv_hashfields_set(dev_flow, rss_desc);
13024         /* If has RSS action in the sample action, the Sample/Mirror resource
13025          * should be registered after the hash filed be update.
13026          */
13027         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
13028                 ret = flow_dv_translate_action_sample(dev,
13029                                                       sample,
13030                                                       dev_flow, attr,
13031                                                       &num_of_dest,
13032                                                       sample_actions,
13033                                                       &sample_res,
13034                                                       error);
13035                 if (ret < 0)
13036                         return ret;
13037                 ret = flow_dv_create_action_sample(dev,
13038                                                    dev_flow,
13039                                                    num_of_dest,
13040                                                    &sample_res,
13041                                                    &mdest_res,
13042                                                    sample_actions,
13043                                                    action_flags,
13044                                                    error);
13045                 if (ret < 0)
13046                         return rte_flow_error_set
13047                                                 (error, rte_errno,
13048                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13049                                                 NULL,
13050                                                 "cannot create sample action");
13051                 if (num_of_dest > 1) {
13052                         dev_flow->dv.actions[sample_act_pos] =
13053                         dev_flow->dv.dest_array_res->action;
13054                 } else {
13055                         dev_flow->dv.actions[sample_act_pos] =
13056                         dev_flow->dv.sample_res->verbs_action;
13057                 }
13058         }
13059         /*
13060          * For multiple destination (sample action with ratio=1), the encap
13061          * action and port id action will be combined into group action.
13062          * So need remove the original these actions in the flow and only
13063          * use the sample action instead of.
13064          */
13065         if (num_of_dest > 1 &&
13066             (sample_act->dr_port_id_action || sample_act->dr_jump_action)) {
13067                 int i;
13068                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
13069
13070                 for (i = 0; i < actions_n; i++) {
13071                         if ((sample_act->dr_encap_action &&
13072                                 sample_act->dr_encap_action ==
13073                                 dev_flow->dv.actions[i]) ||
13074                                 (sample_act->dr_port_id_action &&
13075                                 sample_act->dr_port_id_action ==
13076                                 dev_flow->dv.actions[i]) ||
13077                                 (sample_act->dr_jump_action &&
13078                                 sample_act->dr_jump_action ==
13079                                 dev_flow->dv.actions[i]))
13080                                 continue;
13081                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
13082                 }
13083                 memcpy((void *)dev_flow->dv.actions,
13084                                 (void *)temp_actions,
13085                                 tmp_actions_n * sizeof(void *));
13086                 actions_n = tmp_actions_n;
13087         }
13088         dev_flow->dv.actions_n = actions_n;
13089         dev_flow->act_flags = action_flags;
13090         if (wks->skip_matcher_reg)
13091                 return 0;
13092         /* Register matcher. */
13093         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
13094                                     matcher.mask.size);
13095         matcher.priority = mlx5_get_matcher_priority(dev, attr,
13096                                         matcher.priority);
13097         /* reserved field no needs to be set to 0 here. */
13098         tbl_key.is_fdb = attr->transfer;
13099         tbl_key.is_egress = attr->egress;
13100         tbl_key.level = dev_flow->dv.group;
13101         tbl_key.id = dev_flow->dv.table_id;
13102         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
13103                                      tunnel, attr->group, error))
13104                 return -rte_errno;
13105         return 0;
13106 }
13107
13108 /**
13109  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13110  * and tunnel.
13111  *
13112  * @param[in, out] action
13113  *   Shred RSS action holding hash RX queue objects.
13114  * @param[in] hash_fields
13115  *   Defines combination of packet fields to participate in RX hash.
13116  * @param[in] tunnel
13117  *   Tunnel type
13118  * @param[in] hrxq_idx
13119  *   Hash RX queue index to set.
13120  *
13121  * @return
13122  *   0 on success, otherwise negative errno value.
13123  */
13124 static int
13125 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
13126                               const uint64_t hash_fields,
13127                               uint32_t hrxq_idx)
13128 {
13129         uint32_t *hrxqs = action->hrxq;
13130
13131         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13132         case MLX5_RSS_HASH_IPV4:
13133                 /* fall-through. */
13134         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13135                 /* fall-through. */
13136         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13137                 hrxqs[0] = hrxq_idx;
13138                 return 0;
13139         case MLX5_RSS_HASH_IPV4_TCP:
13140                 /* fall-through. */
13141         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13142                 /* fall-through. */
13143         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13144                 hrxqs[1] = hrxq_idx;
13145                 return 0;
13146         case MLX5_RSS_HASH_IPV4_UDP:
13147                 /* fall-through. */
13148         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13149                 /* fall-through. */
13150         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13151                 hrxqs[2] = hrxq_idx;
13152                 return 0;
13153         case MLX5_RSS_HASH_IPV6:
13154                 /* fall-through. */
13155         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13156                 /* fall-through. */
13157         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13158                 hrxqs[3] = hrxq_idx;
13159                 return 0;
13160         case MLX5_RSS_HASH_IPV6_TCP:
13161                 /* fall-through. */
13162         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13163                 /* fall-through. */
13164         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13165                 hrxqs[4] = hrxq_idx;
13166                 return 0;
13167         case MLX5_RSS_HASH_IPV6_UDP:
13168                 /* fall-through. */
13169         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13170                 /* fall-through. */
13171         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13172                 hrxqs[5] = hrxq_idx;
13173                 return 0;
13174         case MLX5_RSS_HASH_NONE:
13175                 hrxqs[6] = hrxq_idx;
13176                 return 0;
13177         default:
13178                 return -1;
13179         }
13180 }
13181
13182 /**
13183  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13184  * and tunnel.
13185  *
13186  * @param[in] dev
13187  *   Pointer to the Ethernet device structure.
13188  * @param[in] idx
13189  *   Shared RSS action ID holding hash RX queue objects.
13190  * @param[in] hash_fields
13191  *   Defines combination of packet fields to participate in RX hash.
13192  * @param[in] tunnel
13193  *   Tunnel type
13194  *
13195  * @return
13196  *   Valid hash RX queue index, otherwise 0.
13197  */
13198 static uint32_t
13199 __flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
13200                                  const uint64_t hash_fields)
13201 {
13202         struct mlx5_priv *priv = dev->data->dev_private;
13203         struct mlx5_shared_action_rss *shared_rss =
13204             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
13205         const uint32_t *hrxqs = shared_rss->hrxq;
13206
13207         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13208         case MLX5_RSS_HASH_IPV4:
13209                 /* fall-through. */
13210         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13211                 /* fall-through. */
13212         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13213                 return hrxqs[0];
13214         case MLX5_RSS_HASH_IPV4_TCP:
13215                 /* fall-through. */
13216         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13217                 /* fall-through. */
13218         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13219                 return hrxqs[1];
13220         case MLX5_RSS_HASH_IPV4_UDP:
13221                 /* fall-through. */
13222         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13223                 /* fall-through. */
13224         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13225                 return hrxqs[2];
13226         case MLX5_RSS_HASH_IPV6:
13227                 /* fall-through. */
13228         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13229                 /* fall-through. */
13230         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13231                 return hrxqs[3];
13232         case MLX5_RSS_HASH_IPV6_TCP:
13233                 /* fall-through. */
13234         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13235                 /* fall-through. */
13236         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13237                 return hrxqs[4];
13238         case MLX5_RSS_HASH_IPV6_UDP:
13239                 /* fall-through. */
13240         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13241                 /* fall-through. */
13242         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13243                 return hrxqs[5];
13244         case MLX5_RSS_HASH_NONE:
13245                 return hrxqs[6];
13246         default:
13247                 return 0;
13248         }
13249
13250 }
13251
13252 /**
13253  * Apply the flow to the NIC, lock free,
13254  * (mutex should be acquired by caller).
13255  *
13256  * @param[in] dev
13257  *   Pointer to the Ethernet device structure.
13258  * @param[in, out] flow
13259  *   Pointer to flow structure.
13260  * @param[out] error
13261  *   Pointer to error structure.
13262  *
13263  * @return
13264  *   0 on success, a negative errno value otherwise and rte_errno is set.
13265  */
13266 static int
13267 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
13268               struct rte_flow_error *error)
13269 {
13270         struct mlx5_flow_dv_workspace *dv;
13271         struct mlx5_flow_handle *dh;
13272         struct mlx5_flow_handle_dv *dv_h;
13273         struct mlx5_flow *dev_flow;
13274         struct mlx5_priv *priv = dev->data->dev_private;
13275         uint32_t handle_idx;
13276         int n;
13277         int err;
13278         int idx;
13279         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
13280         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
13281
13282         MLX5_ASSERT(wks);
13283         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
13284                 dev_flow = &wks->flows[idx];
13285                 dv = &dev_flow->dv;
13286                 dh = dev_flow->handle;
13287                 dv_h = &dh->dvh;
13288                 n = dv->actions_n;
13289                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
13290                         if (dv->transfer) {
13291                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13292                                 dv->actions[n++] = priv->sh->dr_drop_action;
13293                         } else {
13294 #ifdef HAVE_MLX5DV_DR
13295                                 /* DR supports drop action placeholder. */
13296                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13297                                 dv->actions[n++] = priv->sh->dr_drop_action;
13298 #else
13299                                 /* For DV we use the explicit drop queue. */
13300                                 MLX5_ASSERT(priv->drop_queue.hrxq);
13301                                 dv->actions[n++] =
13302                                                 priv->drop_queue.hrxq->action;
13303 #endif
13304                         }
13305                 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
13306                            !dv_h->rix_sample && !dv_h->rix_dest_array)) {
13307                         struct mlx5_hrxq *hrxq;
13308                         uint32_t hrxq_idx;
13309
13310                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc,
13311                                                     &hrxq_idx);
13312                         if (!hrxq) {
13313                                 rte_flow_error_set
13314                                         (error, rte_errno,
13315                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13316                                          "cannot get hash queue");
13317                                 goto error;
13318                         }
13319                         dh->rix_hrxq = hrxq_idx;
13320                         dv->actions[n++] = hrxq->action;
13321                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13322                         struct mlx5_hrxq *hrxq = NULL;
13323                         uint32_t hrxq_idx;
13324
13325                         hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev,
13326                                                 rss_desc->shared_rss,
13327                                                 dev_flow->hash_fields);
13328                         if (hrxq_idx)
13329                                 hrxq = mlx5_ipool_get
13330                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
13331                                          hrxq_idx);
13332                         if (!hrxq) {
13333                                 rte_flow_error_set
13334                                         (error, rte_errno,
13335                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13336                                          "cannot get hash queue");
13337                                 goto error;
13338                         }
13339                         dh->rix_srss = rss_desc->shared_rss;
13340                         dv->actions[n++] = hrxq->action;
13341                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
13342                         if (!priv->sh->default_miss_action) {
13343                                 rte_flow_error_set
13344                                         (error, rte_errno,
13345                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13346                                          "default miss action not be created.");
13347                                 goto error;
13348                         }
13349                         dv->actions[n++] = priv->sh->default_miss_action;
13350                 }
13351                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
13352                                                (void *)&dv->value, n,
13353                                                dv->actions, &dh->drv_flow);
13354                 if (err) {
13355                         rte_flow_error_set(error, errno,
13356                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
13357                                            NULL,
13358                                            "hardware refuses to create flow");
13359                         goto error;
13360                 }
13361                 if (priv->vmwa_context &&
13362                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
13363                         /*
13364                          * The rule contains the VLAN pattern.
13365                          * For VF we are going to create VLAN
13366                          * interface to make hypervisor set correct
13367                          * e-Switch vport context.
13368                          */
13369                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
13370                 }
13371         }
13372         return 0;
13373 error:
13374         err = rte_errno; /* Save rte_errno before cleanup. */
13375         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
13376                        handle_idx, dh, next) {
13377                 /* hrxq is union, don't clear it if the flag is not set. */
13378                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
13379                         mlx5_hrxq_release(dev, dh->rix_hrxq);
13380                         dh->rix_hrxq = 0;
13381                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13382                         dh->rix_srss = 0;
13383                 }
13384                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
13385                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
13386         }
13387         rte_errno = err; /* Restore rte_errno. */
13388         return -rte_errno;
13389 }
13390
13391 void
13392 flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused,
13393                           struct mlx5_cache_entry *entry)
13394 {
13395         struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache),
13396                                                           entry);
13397
13398         claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object));
13399         mlx5_free(cache);
13400 }
13401
13402 /**
13403  * Release the flow matcher.
13404  *
13405  * @param dev
13406  *   Pointer to Ethernet device.
13407  * @param port_id
13408  *   Index to port ID action resource.
13409  *
13410  * @return
13411  *   1 while a reference on it exists, 0 when freed.
13412  */
13413 static int
13414 flow_dv_matcher_release(struct rte_eth_dev *dev,
13415                         struct mlx5_flow_handle *handle)
13416 {
13417         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
13418         struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl,
13419                                                             typeof(*tbl), tbl);
13420         int ret;
13421
13422         MLX5_ASSERT(matcher->matcher_object);
13423         ret = mlx5_cache_unregister(&tbl->matchers, &matcher->entry);
13424         flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
13425         return ret;
13426 }
13427
13428 /**
13429  * Release encap_decap resource.
13430  *
13431  * @param list
13432  *   Pointer to the hash list.
13433  * @param entry
13434  *   Pointer to exist resource entry object.
13435  */
13436 void
13437 flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
13438                               struct mlx5_hlist_entry *entry)
13439 {
13440         struct mlx5_dev_ctx_shared *sh = list->ctx;
13441         struct mlx5_flow_dv_encap_decap_resource *res =
13442                 container_of(entry, typeof(*res), entry);
13443
13444         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13445         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
13446 }
13447
13448 /**
13449  * Release an encap/decap resource.
13450  *
13451  * @param dev
13452  *   Pointer to Ethernet device.
13453  * @param encap_decap_idx
13454  *   Index of encap decap resource.
13455  *
13456  * @return
13457  *   1 while a reference on it exists, 0 when freed.
13458  */
13459 static int
13460 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
13461                                      uint32_t encap_decap_idx)
13462 {
13463         struct mlx5_priv *priv = dev->data->dev_private;
13464         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
13465
13466         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
13467                                         encap_decap_idx);
13468         if (!cache_resource)
13469                 return 0;
13470         MLX5_ASSERT(cache_resource->action);
13471         return mlx5_hlist_unregister(priv->sh->encaps_decaps,
13472                                      &cache_resource->entry);
13473 }
13474
13475 /**
13476  * Release an jump to table action resource.
13477  *
13478  * @param dev
13479  *   Pointer to Ethernet device.
13480  * @param rix_jump
13481  *   Index to the jump action resource.
13482  *
13483  * @return
13484  *   1 while a reference on it exists, 0 when freed.
13485  */
13486 static int
13487 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
13488                                   uint32_t rix_jump)
13489 {
13490         struct mlx5_priv *priv = dev->data->dev_private;
13491         struct mlx5_flow_tbl_data_entry *tbl_data;
13492
13493         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
13494                                   rix_jump);
13495         if (!tbl_data)
13496                 return 0;
13497         return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl);
13498 }
13499
13500 void
13501 flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused,
13502                          struct mlx5_hlist_entry *entry)
13503 {
13504         struct mlx5_flow_dv_modify_hdr_resource *res =
13505                 container_of(entry, typeof(*res), entry);
13506
13507         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13508         mlx5_free(entry);
13509 }
13510
13511 /**
13512  * Release a modify-header resource.
13513  *
13514  * @param dev
13515  *   Pointer to Ethernet device.
13516  * @param handle
13517  *   Pointer to mlx5_flow_handle.
13518  *
13519  * @return
13520  *   1 while a reference on it exists, 0 when freed.
13521  */
13522 static int
13523 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
13524                                     struct mlx5_flow_handle *handle)
13525 {
13526         struct mlx5_priv *priv = dev->data->dev_private;
13527         struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr;
13528
13529         MLX5_ASSERT(entry->action);
13530         return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry);
13531 }
13532
13533 void
13534 flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
13535                           struct mlx5_cache_entry *entry)
13536 {
13537         struct mlx5_dev_ctx_shared *sh = list->ctx;
13538         struct mlx5_flow_dv_port_id_action_resource *cache =
13539                         container_of(entry, typeof(*cache), entry);
13540
13541         claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
13542         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], cache->idx);
13543 }
13544
13545 /**
13546  * Release port ID action resource.
13547  *
13548  * @param dev
13549  *   Pointer to Ethernet device.
13550  * @param handle
13551  *   Pointer to mlx5_flow_handle.
13552  *
13553  * @return
13554  *   1 while a reference on it exists, 0 when freed.
13555  */
13556 static int
13557 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
13558                                         uint32_t port_id)
13559 {
13560         struct mlx5_priv *priv = dev->data->dev_private;
13561         struct mlx5_flow_dv_port_id_action_resource *cache;
13562
13563         cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
13564         if (!cache)
13565                 return 0;
13566         MLX5_ASSERT(cache->action);
13567         return mlx5_cache_unregister(&priv->sh->port_id_action_list,
13568                                      &cache->entry);
13569 }
13570
13571 /**
13572  * Release shared RSS action resource.
13573  *
13574  * @param dev
13575  *   Pointer to Ethernet device.
13576  * @param srss
13577  *   Shared RSS action index.
13578  */
13579 static void
13580 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
13581 {
13582         struct mlx5_priv *priv = dev->data->dev_private;
13583         struct mlx5_shared_action_rss *shared_rss;
13584
13585         shared_rss = mlx5_ipool_get
13586                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss);
13587         __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
13588 }
13589
13590 void
13591 flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
13592                             struct mlx5_cache_entry *entry)
13593 {
13594         struct mlx5_dev_ctx_shared *sh = list->ctx;
13595         struct mlx5_flow_dv_push_vlan_action_resource *cache =
13596                         container_of(entry, typeof(*cache), entry);
13597
13598         claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
13599         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], cache->idx);
13600 }
13601
13602 /**
13603  * Release push vlan action resource.
13604  *
13605  * @param dev
13606  *   Pointer to Ethernet device.
13607  * @param handle
13608  *   Pointer to mlx5_flow_handle.
13609  *
13610  * @return
13611  *   1 while a reference on it exists, 0 when freed.
13612  */
13613 static int
13614 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
13615                                           struct mlx5_flow_handle *handle)
13616 {
13617         struct mlx5_priv *priv = dev->data->dev_private;
13618         struct mlx5_flow_dv_push_vlan_action_resource *cache;
13619         uint32_t idx = handle->dvh.rix_push_vlan;
13620
13621         cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
13622         if (!cache)
13623                 return 0;
13624         MLX5_ASSERT(cache->action);
13625         return mlx5_cache_unregister(&priv->sh->push_vlan_action_list,
13626                                      &cache->entry);
13627 }
13628
13629 /**
13630  * Release the fate resource.
13631  *
13632  * @param dev
13633  *   Pointer to Ethernet device.
13634  * @param handle
13635  *   Pointer to mlx5_flow_handle.
13636  */
13637 static void
13638 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
13639                                struct mlx5_flow_handle *handle)
13640 {
13641         if (!handle->rix_fate)
13642                 return;
13643         switch (handle->fate_action) {
13644         case MLX5_FLOW_FATE_QUEUE:
13645                 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array)
13646                         mlx5_hrxq_release(dev, handle->rix_hrxq);
13647                 break;
13648         case MLX5_FLOW_FATE_JUMP:
13649                 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump);
13650                 break;
13651         case MLX5_FLOW_FATE_PORT_ID:
13652                 flow_dv_port_id_action_resource_release(dev,
13653                                 handle->rix_port_id_action);
13654                 break;
13655         default:
13656                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
13657                 break;
13658         }
13659         handle->rix_fate = 0;
13660 }
13661
13662 void
13663 flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused,
13664                          struct mlx5_cache_entry *entry)
13665 {
13666         struct mlx5_flow_dv_sample_resource *cache_resource =
13667                         container_of(entry, typeof(*cache_resource), entry);
13668         struct rte_eth_dev *dev = cache_resource->dev;
13669         struct mlx5_priv *priv = dev->data->dev_private;
13670
13671         if (cache_resource->verbs_action)
13672                 claim_zero(mlx5_flow_os_destroy_flow_action
13673                                 (cache_resource->verbs_action));
13674         if (cache_resource->normal_path_tbl)
13675                 flow_dv_tbl_resource_release(MLX5_SH(dev),
13676                         cache_resource->normal_path_tbl);
13677         flow_dv_sample_sub_actions_release(dev,
13678                                 &cache_resource->sample_idx);
13679         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
13680                         cache_resource->idx);
13681         DRV_LOG(DEBUG, "sample resource %p: removed",
13682                 (void *)cache_resource);
13683 }
13684
13685 /**
13686  * Release an sample resource.
13687  *
13688  * @param dev
13689  *   Pointer to Ethernet device.
13690  * @param handle
13691  *   Pointer to mlx5_flow_handle.
13692  *
13693  * @return
13694  *   1 while a reference on it exists, 0 when freed.
13695  */
13696 static int
13697 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
13698                                      struct mlx5_flow_handle *handle)
13699 {
13700         struct mlx5_priv *priv = dev->data->dev_private;
13701         struct mlx5_flow_dv_sample_resource *cache_resource;
13702
13703         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
13704                          handle->dvh.rix_sample);
13705         if (!cache_resource)
13706                 return 0;
13707         MLX5_ASSERT(cache_resource->verbs_action);
13708         return mlx5_cache_unregister(&priv->sh->sample_action_list,
13709                                      &cache_resource->entry);
13710 }
13711
13712 void
13713 flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused,
13714                              struct mlx5_cache_entry *entry)
13715 {
13716         struct mlx5_flow_dv_dest_array_resource *cache_resource =
13717                         container_of(entry, typeof(*cache_resource), entry);
13718         struct rte_eth_dev *dev = cache_resource->dev;
13719         struct mlx5_priv *priv = dev->data->dev_private;
13720         uint32_t i = 0;
13721
13722         MLX5_ASSERT(cache_resource->action);
13723         if (cache_resource->action)
13724                 claim_zero(mlx5_flow_os_destroy_flow_action
13725                                         (cache_resource->action));
13726         for (; i < cache_resource->num_of_dest; i++)
13727                 flow_dv_sample_sub_actions_release(dev,
13728                                 &cache_resource->sample_idx[i]);
13729         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
13730                         cache_resource->idx);
13731         DRV_LOG(DEBUG, "destination array resource %p: removed",
13732                 (void *)cache_resource);
13733 }
13734
13735 /**
13736  * Release an destination array resource.
13737  *
13738  * @param dev
13739  *   Pointer to Ethernet device.
13740  * @param handle
13741  *   Pointer to mlx5_flow_handle.
13742  *
13743  * @return
13744  *   1 while a reference on it exists, 0 when freed.
13745  */
13746 static int
13747 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
13748                                     struct mlx5_flow_handle *handle)
13749 {
13750         struct mlx5_priv *priv = dev->data->dev_private;
13751         struct mlx5_flow_dv_dest_array_resource *cache;
13752
13753         cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
13754                                handle->dvh.rix_dest_array);
13755         if (!cache)
13756                 return 0;
13757         MLX5_ASSERT(cache->action);
13758         return mlx5_cache_unregister(&priv->sh->dest_array_list,
13759                                      &cache->entry);
13760 }
13761
13762 static void
13763 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev)
13764 {
13765         struct mlx5_priv *priv = dev->data->dev_private;
13766         struct mlx5_dev_ctx_shared *sh = priv->sh;
13767         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
13768                                 sh->geneve_tlv_option_resource;
13769         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
13770         if (geneve_opt_resource) {
13771                 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1,
13772                                          __ATOMIC_RELAXED))) {
13773                         claim_zero(mlx5_devx_cmd_destroy
13774                                         (geneve_opt_resource->obj));
13775                         mlx5_free(sh->geneve_tlv_option_resource);
13776                         sh->geneve_tlv_option_resource = NULL;
13777                 }
13778         }
13779         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
13780 }
13781
13782 /**
13783  * Remove the flow from the NIC but keeps it in memory.
13784  * Lock free, (mutex should be acquired by caller).
13785  *
13786  * @param[in] dev
13787  *   Pointer to Ethernet device.
13788  * @param[in, out] flow
13789  *   Pointer to flow structure.
13790  */
13791 static void
13792 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
13793 {
13794         struct mlx5_flow_handle *dh;
13795         uint32_t handle_idx;
13796         struct mlx5_priv *priv = dev->data->dev_private;
13797
13798         if (!flow)
13799                 return;
13800         handle_idx = flow->dev_handles;
13801         while (handle_idx) {
13802                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
13803                                     handle_idx);
13804                 if (!dh)
13805                         return;
13806                 if (dh->drv_flow) {
13807                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
13808                         dh->drv_flow = NULL;
13809                 }
13810                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
13811                         flow_dv_fate_resource_release(dev, dh);
13812                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
13813                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
13814                 handle_idx = dh->next.next;
13815         }
13816 }
13817
13818 /**
13819  * Remove the flow from the NIC and the memory.
13820  * Lock free, (mutex should be acquired by caller).
13821  *
13822  * @param[in] dev
13823  *   Pointer to the Ethernet device structure.
13824  * @param[in, out] flow
13825  *   Pointer to flow structure.
13826  */
13827 static void
13828 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
13829 {
13830         struct mlx5_flow_handle *dev_handle;
13831         struct mlx5_priv *priv = dev->data->dev_private;
13832         struct mlx5_flow_meter_info *fm = NULL;
13833         uint32_t srss = 0;
13834
13835         if (!flow)
13836                 return;
13837         flow_dv_remove(dev, flow);
13838         if (flow->counter) {
13839                 flow_dv_counter_free(dev, flow->counter);
13840                 flow->counter = 0;
13841         }
13842         if (flow->meter) {
13843                 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
13844                 if (fm)
13845                         mlx5_flow_meter_detach(priv, fm);
13846                 flow->meter = 0;
13847         }
13848         /* Keep the current age handling by default. */
13849         if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct)
13850                 flow_dv_aso_ct_release(dev, flow->ct);
13851         else if (flow->age)
13852                 flow_dv_aso_age_release(dev, flow->age);
13853         if (flow->geneve_tlv_option) {
13854                 flow_dv_geneve_tlv_option_resource_release(dev);
13855                 flow->geneve_tlv_option = 0;
13856         }
13857         while (flow->dev_handles) {
13858                 uint32_t tmp_idx = flow->dev_handles;
13859
13860                 dev_handle = mlx5_ipool_get(priv->sh->ipool
13861                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
13862                 if (!dev_handle)
13863                         return;
13864                 flow->dev_handles = dev_handle->next.next;
13865                 if (dev_handle->dvh.matcher)
13866                         flow_dv_matcher_release(dev, dev_handle);
13867                 if (dev_handle->dvh.rix_sample)
13868                         flow_dv_sample_resource_release(dev, dev_handle);
13869                 if (dev_handle->dvh.rix_dest_array)
13870                         flow_dv_dest_array_resource_release(dev, dev_handle);
13871                 if (dev_handle->dvh.rix_encap_decap)
13872                         flow_dv_encap_decap_resource_release(dev,
13873                                 dev_handle->dvh.rix_encap_decap);
13874                 if (dev_handle->dvh.modify_hdr)
13875                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
13876                 if (dev_handle->dvh.rix_push_vlan)
13877                         flow_dv_push_vlan_action_resource_release(dev,
13878                                                                   dev_handle);
13879                 if (dev_handle->dvh.rix_tag)
13880                         flow_dv_tag_release(dev,
13881                                             dev_handle->dvh.rix_tag);
13882                 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS)
13883                         flow_dv_fate_resource_release(dev, dev_handle);
13884                 else if (!srss)
13885                         srss = dev_handle->rix_srss;
13886                 if (fm && dev_handle->is_meter_flow_id &&
13887                     dev_handle->split_flow_id)
13888                         mlx5_ipool_free(fm->flow_ipool,
13889                                         dev_handle->split_flow_id);
13890                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
13891                            tmp_idx);
13892         }
13893         if (srss)
13894                 flow_dv_shared_rss_action_release(dev, srss);
13895 }
13896
13897 /**
13898  * Release array of hash RX queue objects.
13899  * Helper function.
13900  *
13901  * @param[in] dev
13902  *   Pointer to the Ethernet device structure.
13903  * @param[in, out] hrxqs
13904  *   Array of hash RX queue objects.
13905  *
13906  * @return
13907  *   Total number of references to hash RX queue objects in *hrxqs* array
13908  *   after this operation.
13909  */
13910 static int
13911 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
13912                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
13913 {
13914         size_t i;
13915         int remaining = 0;
13916
13917         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
13918                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
13919
13920                 if (!ret)
13921                         (*hrxqs)[i] = 0;
13922                 remaining += ret;
13923         }
13924         return remaining;
13925 }
13926
13927 /**
13928  * Release all hash RX queue objects representing shared RSS action.
13929  *
13930  * @param[in] dev
13931  *   Pointer to the Ethernet device structure.
13932  * @param[in, out] action
13933  *   Shared RSS action to remove hash RX queue objects from.
13934  *
13935  * @return
13936  *   Total number of references to hash RX queue objects stored in *action*
13937  *   after this operation.
13938  *   Expected to be 0 if no external references held.
13939  */
13940 static int
13941 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
13942                                  struct mlx5_shared_action_rss *shared_rss)
13943 {
13944         return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq);
13945 }
13946
13947 /**
13948  * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to
13949  * user input.
13950  *
13951  * Only one hash value is available for one L3+L4 combination:
13952  * for example:
13953  * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and
13954  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
13955  * same slot in mlx5_rss_hash_fields.
13956  *
13957  * @param[in] rss
13958  *   Pointer to the shared action RSS conf.
13959  * @param[in, out] hash_field
13960  *   hash_field variable needed to be adjusted.
13961  *
13962  * @return
13963  *   void
13964  */
13965 static void
13966 __flow_dv_action_rss_l34_hash_adjust(struct mlx5_shared_action_rss *rss,
13967                                      uint64_t *hash_field)
13968 {
13969         uint64_t rss_types = rss->origin.types;
13970
13971         switch (*hash_field & ~IBV_RX_HASH_INNER) {
13972         case MLX5_RSS_HASH_IPV4:
13973                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
13974                         *hash_field &= ~MLX5_RSS_HASH_IPV4;
13975                         if (rss_types & ETH_RSS_L3_DST_ONLY)
13976                                 *hash_field |= IBV_RX_HASH_DST_IPV4;
13977                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
13978                                 *hash_field |= IBV_RX_HASH_SRC_IPV4;
13979                         else
13980                                 *hash_field |= MLX5_RSS_HASH_IPV4;
13981                 }
13982                 return;
13983         case MLX5_RSS_HASH_IPV6:
13984                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
13985                         *hash_field &= ~MLX5_RSS_HASH_IPV6;
13986                         if (rss_types & ETH_RSS_L3_DST_ONLY)
13987                                 *hash_field |= IBV_RX_HASH_DST_IPV6;
13988                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
13989                                 *hash_field |= IBV_RX_HASH_SRC_IPV6;
13990                         else
13991                                 *hash_field |= MLX5_RSS_HASH_IPV6;
13992                 }
13993                 return;
13994         case MLX5_RSS_HASH_IPV4_UDP:
13995                 /* fall-through. */
13996         case MLX5_RSS_HASH_IPV6_UDP:
13997                 if (rss_types & ETH_RSS_UDP) {
13998                         *hash_field &= ~MLX5_UDP_IBV_RX_HASH;
13999                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14000                                 *hash_field |= IBV_RX_HASH_DST_PORT_UDP;
14001                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14002                                 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP;
14003                         else
14004                                 *hash_field |= MLX5_UDP_IBV_RX_HASH;
14005                 }
14006                 return;
14007         case MLX5_RSS_HASH_IPV4_TCP:
14008                 /* fall-through. */
14009         case MLX5_RSS_HASH_IPV6_TCP:
14010                 if (rss_types & ETH_RSS_TCP) {
14011                         *hash_field &= ~MLX5_TCP_IBV_RX_HASH;
14012                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14013                                 *hash_field |= IBV_RX_HASH_DST_PORT_TCP;
14014                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14015                                 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP;
14016                         else
14017                                 *hash_field |= MLX5_TCP_IBV_RX_HASH;
14018                 }
14019                 return;
14020         default:
14021                 return;
14022         }
14023 }
14024
14025 /**
14026  * Setup shared RSS action.
14027  * Prepare set of hash RX queue objects sufficient to handle all valid
14028  * hash_fields combinations (see enum ibv_rx_hash_fields).
14029  *
14030  * @param[in] dev
14031  *   Pointer to the Ethernet device structure.
14032  * @param[in] action_idx
14033  *   Shared RSS action ipool index.
14034  * @param[in, out] action
14035  *   Partially initialized shared RSS action.
14036  * @param[out] error
14037  *   Perform verbose error reporting if not NULL. Initialized in case of
14038  *   error only.
14039  *
14040  * @return
14041  *   0 on success, otherwise negative errno value.
14042  */
14043 static int
14044 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
14045                            uint32_t action_idx,
14046                            struct mlx5_shared_action_rss *shared_rss,
14047                            struct rte_flow_error *error)
14048 {
14049         struct mlx5_flow_rss_desc rss_desc = { 0 };
14050         size_t i;
14051         int err;
14052
14053         if (mlx5_ind_table_obj_setup(dev, shared_rss->ind_tbl)) {
14054                 return rte_flow_error_set(error, rte_errno,
14055                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14056                                           "cannot setup indirection table");
14057         }
14058         memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN);
14059         rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
14060         rss_desc.const_q = shared_rss->origin.queue;
14061         rss_desc.queue_num = shared_rss->origin.queue_num;
14062         /* Set non-zero value to indicate a shared RSS. */
14063         rss_desc.shared_rss = action_idx;
14064         rss_desc.ind_tbl = shared_rss->ind_tbl;
14065         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
14066                 uint32_t hrxq_idx;
14067                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
14068                 int tunnel = 0;
14069
14070                 __flow_dv_action_rss_l34_hash_adjust(shared_rss, &hash_fields);
14071                 if (shared_rss->origin.level > 1) {
14072                         hash_fields |= IBV_RX_HASH_INNER;
14073                         tunnel = 1;
14074                 }
14075                 rss_desc.tunnel = tunnel;
14076                 rss_desc.hash_fields = hash_fields;
14077                 hrxq_idx = mlx5_hrxq_get(dev, &rss_desc);
14078                 if (!hrxq_idx) {
14079                         rte_flow_error_set
14080                                 (error, rte_errno,
14081                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14082                                  "cannot get hash queue");
14083                         goto error_hrxq_new;
14084                 }
14085                 err = __flow_dv_action_rss_hrxq_set
14086                         (shared_rss, hash_fields, hrxq_idx);
14087                 MLX5_ASSERT(!err);
14088         }
14089         return 0;
14090 error_hrxq_new:
14091         err = rte_errno;
14092         __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14093         if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true))
14094                 shared_rss->ind_tbl = NULL;
14095         rte_errno = err;
14096         return -rte_errno;
14097 }
14098
14099 /**
14100  * Create shared RSS action.
14101  *
14102  * @param[in] dev
14103  *   Pointer to the Ethernet device structure.
14104  * @param[in] conf
14105  *   Shared action configuration.
14106  * @param[in] rss
14107  *   RSS action specification used to create shared action.
14108  * @param[out] error
14109  *   Perform verbose error reporting if not NULL. Initialized in case of
14110  *   error only.
14111  *
14112  * @return
14113  *   A valid shared action ID in case of success, 0 otherwise and
14114  *   rte_errno is set.
14115  */
14116 static uint32_t
14117 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
14118                             const struct rte_flow_indir_action_conf *conf,
14119                             const struct rte_flow_action_rss *rss,
14120                             struct rte_flow_error *error)
14121 {
14122         struct mlx5_priv *priv = dev->data->dev_private;
14123         struct mlx5_shared_action_rss *shared_rss = NULL;
14124         void *queue = NULL;
14125         struct rte_flow_action_rss *origin;
14126         const uint8_t *rss_key;
14127         uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
14128         uint32_t idx;
14129
14130         RTE_SET_USED(conf);
14131         queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14132                             0, SOCKET_ID_ANY);
14133         shared_rss = mlx5_ipool_zmalloc
14134                          (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx);
14135         if (!shared_rss || !queue) {
14136                 rte_flow_error_set(error, ENOMEM,
14137                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14138                                    "cannot allocate resource memory");
14139                 goto error_rss_init;
14140         }
14141         if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) {
14142                 rte_flow_error_set(error, E2BIG,
14143                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14144                                    "rss action number out of range");
14145                 goto error_rss_init;
14146         }
14147         shared_rss->ind_tbl = mlx5_malloc(MLX5_MEM_ZERO,
14148                                           sizeof(*shared_rss->ind_tbl),
14149                                           0, SOCKET_ID_ANY);
14150         if (!shared_rss->ind_tbl) {
14151                 rte_flow_error_set(error, ENOMEM,
14152                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14153                                    "cannot allocate resource memory");
14154                 goto error_rss_init;
14155         }
14156         memcpy(queue, rss->queue, queue_size);
14157         shared_rss->ind_tbl->queues = queue;
14158         shared_rss->ind_tbl->queues_n = rss->queue_num;
14159         origin = &shared_rss->origin;
14160         origin->func = rss->func;
14161         origin->level = rss->level;
14162         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
14163         origin->types = !rss->types ? ETH_RSS_IP : rss->types;
14164         /* NULL RSS key indicates default RSS key. */
14165         rss_key = !rss->key ? rss_hash_default_key : rss->key;
14166         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
14167         origin->key = &shared_rss->key[0];
14168         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
14169         origin->queue = queue;
14170         origin->queue_num = rss->queue_num;
14171         if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error))
14172                 goto error_rss_init;
14173         rte_spinlock_init(&shared_rss->action_rss_sl);
14174         __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14175         rte_spinlock_lock(&priv->shared_act_sl);
14176         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14177                      &priv->rss_shared_actions, idx, shared_rss, next);
14178         rte_spinlock_unlock(&priv->shared_act_sl);
14179         return idx;
14180 error_rss_init:
14181         if (shared_rss) {
14182                 if (shared_rss->ind_tbl)
14183                         mlx5_free(shared_rss->ind_tbl);
14184                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14185                                 idx);
14186         }
14187         if (queue)
14188                 mlx5_free(queue);
14189         return 0;
14190 }
14191
14192 /**
14193  * Destroy the shared RSS action.
14194  * Release related hash RX queue objects.
14195  *
14196  * @param[in] dev
14197  *   Pointer to the Ethernet device structure.
14198  * @param[in] idx
14199  *   The shared RSS action object ID to be removed.
14200  * @param[out] error
14201  *   Perform verbose error reporting if not NULL. Initialized in case of
14202  *   error only.
14203  *
14204  * @return
14205  *   0 on success, otherwise negative errno value.
14206  */
14207 static int
14208 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx,
14209                              struct rte_flow_error *error)
14210 {
14211         struct mlx5_priv *priv = dev->data->dev_private;
14212         struct mlx5_shared_action_rss *shared_rss =
14213             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14214         uint32_t old_refcnt = 1;
14215         int remaining;
14216         uint16_t *queue = NULL;
14217
14218         if (!shared_rss)
14219                 return rte_flow_error_set(error, EINVAL,
14220                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14221                                           "invalid shared action");
14222         remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14223         if (remaining)
14224                 return rte_flow_error_set(error, EBUSY,
14225                                           RTE_FLOW_ERROR_TYPE_ACTION,
14226                                           NULL,
14227                                           "shared rss hrxq has references");
14228         if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt,
14229                                          0, 0, __ATOMIC_ACQUIRE,
14230                                          __ATOMIC_RELAXED))
14231                 return rte_flow_error_set(error, EBUSY,
14232                                           RTE_FLOW_ERROR_TYPE_ACTION,
14233                                           NULL,
14234                                           "shared rss has references");
14235         queue = shared_rss->ind_tbl->queues;
14236         remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true);
14237         if (remaining)
14238                 return rte_flow_error_set(error, EBUSY,
14239                                           RTE_FLOW_ERROR_TYPE_ACTION,
14240                                           NULL,
14241                                           "shared rss indirection table has"
14242                                           " references");
14243         mlx5_free(queue);
14244         rte_spinlock_lock(&priv->shared_act_sl);
14245         ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14246                      &priv->rss_shared_actions, idx, shared_rss, next);
14247         rte_spinlock_unlock(&priv->shared_act_sl);
14248         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14249                         idx);
14250         return 0;
14251 }
14252
14253 /**
14254  * Create indirect action, lock free,
14255  * (mutex should be acquired by caller).
14256  * Dispatcher for action type specific call.
14257  *
14258  * @param[in] dev
14259  *   Pointer to the Ethernet device structure.
14260  * @param[in] conf
14261  *   Shared action configuration.
14262  * @param[in] action
14263  *   Action specification used to create indirect action.
14264  * @param[out] error
14265  *   Perform verbose error reporting if not NULL. Initialized in case of
14266  *   error only.
14267  *
14268  * @return
14269  *   A valid shared action handle in case of success, NULL otherwise and
14270  *   rte_errno is set.
14271  */
14272 static struct rte_flow_action_handle *
14273 flow_dv_action_create(struct rte_eth_dev *dev,
14274                       const struct rte_flow_indir_action_conf *conf,
14275                       const struct rte_flow_action *action,
14276                       struct rte_flow_error *err)
14277 {
14278         struct mlx5_priv *priv = dev->data->dev_private;
14279         uint32_t age_idx = 0;
14280         uint32_t idx = 0;
14281         uint32_t ret = 0;
14282
14283         switch (action->type) {
14284         case RTE_FLOW_ACTION_TYPE_RSS:
14285                 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err);
14286                 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS <<
14287                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14288                 break;
14289         case RTE_FLOW_ACTION_TYPE_AGE:
14290                 age_idx = flow_dv_aso_age_alloc(dev, err);
14291                 if (!age_idx) {
14292                         ret = -rte_errno;
14293                         break;
14294                 }
14295                 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
14296                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
14297                 flow_dv_aso_age_params_init(dev, age_idx,
14298                                         ((const struct rte_flow_action_age *)
14299                                                 action->conf)->context ?
14300                                         ((const struct rte_flow_action_age *)
14301                                                 action->conf)->context :
14302                                         (void *)(uintptr_t)idx,
14303                                         ((const struct rte_flow_action_age *)
14304                                                 action->conf)->timeout);
14305                 ret = age_idx;
14306                 break;
14307         case RTE_FLOW_ACTION_TYPE_COUNT:
14308                 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL);
14309                 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT <<
14310                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14311                 break;
14312         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
14313                 ret = flow_dv_translate_create_conntrack(dev, action->conf,
14314                                                          err);
14315                 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret);
14316                 break;
14317         default:
14318                 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
14319                                    NULL, "action type not supported");
14320                 break;
14321         }
14322         return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL;
14323 }
14324
14325 /**
14326  * Destroy the indirect action.
14327  * Release action related resources on the NIC and the memory.
14328  * Lock free, (mutex should be acquired by caller).
14329  * Dispatcher for action type specific call.
14330  *
14331  * @param[in] dev
14332  *   Pointer to the Ethernet device structure.
14333  * @param[in] handle
14334  *   The indirect action object handle to be removed.
14335  * @param[out] error
14336  *   Perform verbose error reporting if not NULL. Initialized in case of
14337  *   error only.
14338  *
14339  * @return
14340  *   0 on success, otherwise negative errno value.
14341  */
14342 static int
14343 flow_dv_action_destroy(struct rte_eth_dev *dev,
14344                        struct rte_flow_action_handle *handle,
14345                        struct rte_flow_error *error)
14346 {
14347         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14348         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14349         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14350         struct mlx5_flow_counter *cnt;
14351         uint32_t no_flow_refcnt = 1;
14352         int ret;
14353
14354         switch (type) {
14355         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14356                 return __flow_dv_action_rss_release(dev, idx, error);
14357         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
14358                 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL);
14359                 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt,
14360                                                  &no_flow_refcnt, 1, false,
14361                                                  __ATOMIC_ACQUIRE,
14362                                                  __ATOMIC_RELAXED))
14363                         return rte_flow_error_set(error, EBUSY,
14364                                                   RTE_FLOW_ERROR_TYPE_ACTION,
14365                                                   NULL,
14366                                                   "Indirect count action has references");
14367                 flow_dv_counter_free(dev, idx);
14368                 return 0;
14369         case MLX5_INDIRECT_ACTION_TYPE_AGE:
14370                 ret = flow_dv_aso_age_release(dev, idx);
14371                 if (ret)
14372                         /*
14373                          * In this case, the last flow has a reference will
14374                          * actually release the age action.
14375                          */
14376                         DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was"
14377                                 " released with references %d.", idx, ret);
14378                 return 0;
14379         case MLX5_INDIRECT_ACTION_TYPE_CT:
14380                 ret = flow_dv_aso_ct_release(dev, idx);
14381                 if (ret < 0)
14382                         return ret;
14383                 if (ret > 0)
14384                         DRV_LOG(DEBUG, "Connection tracking object %u still "
14385                                 "has references %d.", idx, ret);
14386                 return 0;
14387         default:
14388                 return rte_flow_error_set(error, ENOTSUP,
14389                                           RTE_FLOW_ERROR_TYPE_ACTION,
14390                                           NULL,
14391                                           "action type not supported");
14392         }
14393 }
14394
14395 /**
14396  * Updates in place shared RSS action configuration.
14397  *
14398  * @param[in] dev
14399  *   Pointer to the Ethernet device structure.
14400  * @param[in] idx
14401  *   The shared RSS action object ID to be updated.
14402  * @param[in] action_conf
14403  *   RSS action specification used to modify *shared_rss*.
14404  * @param[out] error
14405  *   Perform verbose error reporting if not NULL. Initialized in case of
14406  *   error only.
14407  *
14408  * @return
14409  *   0 on success, otherwise negative errno value.
14410  * @note: currently only support update of RSS queues.
14411  */
14412 static int
14413 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx,
14414                             const struct rte_flow_action_rss *action_conf,
14415                             struct rte_flow_error *error)
14416 {
14417         struct mlx5_priv *priv = dev->data->dev_private;
14418         struct mlx5_shared_action_rss *shared_rss =
14419             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14420         int ret = 0;
14421         void *queue = NULL;
14422         uint16_t *queue_old = NULL;
14423         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
14424
14425         if (!shared_rss)
14426                 return rte_flow_error_set(error, EINVAL,
14427                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14428                                           "invalid shared action to update");
14429         if (priv->obj_ops.ind_table_modify == NULL)
14430                 return rte_flow_error_set(error, ENOTSUP,
14431                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14432                                           "cannot modify indirection table");
14433         queue = mlx5_malloc(MLX5_MEM_ZERO,
14434                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14435                             0, SOCKET_ID_ANY);
14436         if (!queue)
14437                 return rte_flow_error_set(error, ENOMEM,
14438                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14439                                           NULL,
14440                                           "cannot allocate resource memory");
14441         memcpy(queue, action_conf->queue, queue_size);
14442         MLX5_ASSERT(shared_rss->ind_tbl);
14443         rte_spinlock_lock(&shared_rss->action_rss_sl);
14444         queue_old = shared_rss->ind_tbl->queues;
14445         ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl,
14446                                         queue, action_conf->queue_num, true);
14447         if (ret) {
14448                 mlx5_free(queue);
14449                 ret = rte_flow_error_set(error, rte_errno,
14450                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14451                                           "cannot update indirection table");
14452         } else {
14453                 mlx5_free(queue_old);
14454                 shared_rss->origin.queue = queue;
14455                 shared_rss->origin.queue_num = action_conf->queue_num;
14456         }
14457         rte_spinlock_unlock(&shared_rss->action_rss_sl);
14458         return ret;
14459 }
14460
14461 /*
14462  * Updates in place conntrack context or direction.
14463  * Context update should be synchronized.
14464  *
14465  * @param[in] dev
14466  *   Pointer to the Ethernet device structure.
14467  * @param[in] idx
14468  *   The conntrack object ID to be updated.
14469  * @param[in] update
14470  *   Pointer to the structure of information to update.
14471  * @param[out] error
14472  *   Perform verbose error reporting if not NULL. Initialized in case of
14473  *   error only.
14474  *
14475  * @return
14476  *   0 on success, otherwise negative errno value.
14477  */
14478 static int
14479 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx,
14480                            const struct rte_flow_modify_conntrack *update,
14481                            struct rte_flow_error *error)
14482 {
14483         struct mlx5_priv *priv = dev->data->dev_private;
14484         struct mlx5_aso_ct_action *ct;
14485         const struct rte_flow_action_conntrack *new_prf;
14486         int ret = 0;
14487         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
14488         uint32_t dev_idx;
14489
14490         if (PORT_ID(priv) != owner)
14491                 return rte_flow_error_set(error, EACCES,
14492                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14493                                           NULL,
14494                                           "CT object owned by another port");
14495         dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
14496         ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
14497         if (!ct->refcnt)
14498                 return rte_flow_error_set(error, ENOMEM,
14499                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14500                                           NULL,
14501                                           "CT object is inactive");
14502         new_prf = &update->new_ct;
14503         if (update->direction)
14504                 ct->is_original = !!new_prf->is_original_dir;
14505         if (update->state) {
14506                 /* Only validate the profile when it needs to be updated. */
14507                 ret = mlx5_validate_action_ct(dev, new_prf, error);
14508                 if (ret)
14509                         return ret;
14510                 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf);
14511                 if (ret)
14512                         return rte_flow_error_set(error, EIO,
14513                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14514                                         NULL,
14515                                         "Failed to send CT context update WQE");
14516                 /* Block until ready or a failure. */
14517                 ret = mlx5_aso_ct_available(priv->sh, ct);
14518                 if (ret)
14519                         rte_flow_error_set(error, rte_errno,
14520                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14521                                            NULL,
14522                                            "Timeout to get the CT update");
14523         }
14524         return ret;
14525 }
14526
14527 /**
14528  * Updates in place shared action configuration, lock free,
14529  * (mutex should be acquired by caller).
14530  *
14531  * @param[in] dev
14532  *   Pointer to the Ethernet device structure.
14533  * @param[in] handle
14534  *   The indirect action object handle to be updated.
14535  * @param[in] update
14536  *   Action specification used to modify the action pointed by *handle*.
14537  *   *update* could be of same type with the action pointed by the *handle*
14538  *   handle argument, or some other structures like a wrapper, depending on
14539  *   the indirect action type.
14540  * @param[out] error
14541  *   Perform verbose error reporting if not NULL. Initialized in case of
14542  *   error only.
14543  *
14544  * @return
14545  *   0 on success, otherwise negative errno value.
14546  */
14547 static int
14548 flow_dv_action_update(struct rte_eth_dev *dev,
14549                         struct rte_flow_action_handle *handle,
14550                         const void *update,
14551                         struct rte_flow_error *err)
14552 {
14553         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14554         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14555         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14556         const void *action_conf;
14557
14558         switch (type) {
14559         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14560                 action_conf = ((const struct rte_flow_action *)update)->conf;
14561                 return __flow_dv_action_rss_update(dev, idx, action_conf, err);
14562         case MLX5_INDIRECT_ACTION_TYPE_CT:
14563                 return __flow_dv_action_ct_update(dev, idx, update, err);
14564         default:
14565                 return rte_flow_error_set(err, ENOTSUP,
14566                                           RTE_FLOW_ERROR_TYPE_ACTION,
14567                                           NULL,
14568                                           "action type update not supported");
14569         }
14570 }
14571
14572 /**
14573  * Destroy the meter sub policy table rules.
14574  * Lock free, (mutex should be acquired by caller).
14575  *
14576  * @param[in] dev
14577  *   Pointer to Ethernet device.
14578  * @param[in] sub_policy
14579  *   Pointer to meter sub policy table.
14580  */
14581 static void
14582 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
14583                              struct mlx5_flow_meter_sub_policy *sub_policy)
14584 {
14585         struct mlx5_flow_tbl_data_entry *tbl;
14586         int i;
14587
14588         for (i = 0; i < RTE_COLORS; i++) {
14589                 if (sub_policy->color_rule[i]) {
14590                         claim_zero(mlx5_flow_os_destroy_flow
14591                                 (sub_policy->color_rule[i]));
14592                         sub_policy->color_rule[i] = NULL;
14593                 }
14594                 if (sub_policy->color_matcher[i]) {
14595                         tbl = container_of(sub_policy->color_matcher[i]->tbl,
14596                                 typeof(*tbl), tbl);
14597                         mlx5_cache_unregister(&tbl->matchers,
14598                                       &sub_policy->color_matcher[i]->entry);
14599                         sub_policy->color_matcher[i] = NULL;
14600                 }
14601         }
14602         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
14603                 if (sub_policy->rix_hrxq[i]) {
14604                         mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]);
14605                         sub_policy->rix_hrxq[i] = 0;
14606                 }
14607                 if (sub_policy->jump_tbl[i]) {
14608                         flow_dv_tbl_resource_release(MLX5_SH(dev),
14609                         sub_policy->jump_tbl[i]);
14610                         sub_policy->jump_tbl[i] = NULL;
14611                 }
14612         }
14613         if (sub_policy->tbl_rsc) {
14614                 flow_dv_tbl_resource_release(MLX5_SH(dev),
14615                         sub_policy->tbl_rsc);
14616                 sub_policy->tbl_rsc = NULL;
14617         }
14618 }
14619
14620 /**
14621  * Destroy policy rules, lock free,
14622  * (mutex should be acquired by caller).
14623  * Dispatcher for action type specific call.
14624  *
14625  * @param[in] dev
14626  *   Pointer to the Ethernet device structure.
14627  * @param[in] mtr_policy
14628  *   Meter policy struct.
14629  */
14630 static void
14631 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev,
14632                       struct mlx5_flow_meter_policy *mtr_policy)
14633 {
14634         uint32_t i, j;
14635         struct mlx5_flow_meter_sub_policy *sub_policy;
14636         uint16_t sub_policy_num;
14637
14638         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
14639                 sub_policy_num = (mtr_policy->sub_policy_num >>
14640                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
14641                         MLX5_MTR_SUB_POLICY_NUM_MASK;
14642                 for (j = 0; j < sub_policy_num; j++) {
14643                         sub_policy = mtr_policy->sub_policys[i][j];
14644                         if (sub_policy)
14645                                 __flow_dv_destroy_sub_policy_rules
14646                                                 (dev, sub_policy);
14647                 }
14648         }
14649 }
14650
14651 /**
14652  * Destroy policy action, lock free,
14653  * (mutex should be acquired by caller).
14654  * Dispatcher for action type specific call.
14655  *
14656  * @param[in] dev
14657  *   Pointer to the Ethernet device structure.
14658  * @param[in] mtr_policy
14659  *   Meter policy struct.
14660  */
14661 static void
14662 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
14663                       struct mlx5_flow_meter_policy *mtr_policy)
14664 {
14665         struct rte_flow_action *rss_action;
14666         struct mlx5_flow_handle dev_handle;
14667         uint32_t i, j;
14668
14669         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
14670                 if (mtr_policy->act_cnt[i].rix_mark) {
14671                         flow_dv_tag_release(dev,
14672                                 mtr_policy->act_cnt[i].rix_mark);
14673                         mtr_policy->act_cnt[i].rix_mark = 0;
14674                 }
14675                 if (mtr_policy->act_cnt[i].modify_hdr) {
14676                         dev_handle.dvh.modify_hdr =
14677                                 mtr_policy->act_cnt[i].modify_hdr;
14678                         flow_dv_modify_hdr_resource_release(dev, &dev_handle);
14679                 }
14680                 switch (mtr_policy->act_cnt[i].fate_action) {
14681                 case MLX5_FLOW_FATE_SHARED_RSS:
14682                         rss_action = mtr_policy->act_cnt[i].rss;
14683                         mlx5_free(rss_action);
14684                         break;
14685                 case MLX5_FLOW_FATE_PORT_ID:
14686                         if (mtr_policy->act_cnt[i].rix_port_id_action) {
14687                                 flow_dv_port_id_action_resource_release(dev,
14688                                 mtr_policy->act_cnt[i].rix_port_id_action);
14689                                 mtr_policy->act_cnt[i].rix_port_id_action = 0;
14690                         }
14691                         break;
14692                 case MLX5_FLOW_FATE_DROP:
14693                 case MLX5_FLOW_FATE_JUMP:
14694                         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
14695                                 mtr_policy->act_cnt[i].dr_jump_action[j] =
14696                                                 NULL;
14697                         break;
14698                 default:
14699                         /*Queue action do nothing*/
14700                         break;
14701                 }
14702         }
14703         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
14704                 mtr_policy->dr_drop_action[j] = NULL;
14705 }
14706
14707 /**
14708  * Create policy action per domain, lock free,
14709  * (mutex should be acquired by caller).
14710  * Dispatcher for action type specific call.
14711  *
14712  * @param[in] dev
14713  *   Pointer to the Ethernet device structure.
14714  * @param[in] mtr_policy
14715  *   Meter policy struct.
14716  * @param[in] action
14717  *   Action specification used to create meter actions.
14718  * @param[out] error
14719  *   Perform verbose error reporting if not NULL. Initialized in case of
14720  *   error only.
14721  *
14722  * @return
14723  *   0 on success, otherwise negative errno value.
14724  */
14725 static int
14726 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
14727                         struct mlx5_flow_meter_policy *mtr_policy,
14728                         const struct rte_flow_action *actions[RTE_COLORS],
14729                         enum mlx5_meter_domain domain,
14730                         struct rte_mtr_error *error)
14731 {
14732         struct mlx5_priv *priv = dev->data->dev_private;
14733         struct rte_flow_error flow_err;
14734         const struct rte_flow_action *act;
14735         uint64_t action_flags = 0;
14736         struct mlx5_flow_handle dh;
14737         struct mlx5_flow dev_flow;
14738         struct mlx5_flow_dv_port_id_action_resource port_id_action;
14739         int i, ret;
14740         uint8_t egress, transfer;
14741         struct mlx5_meter_policy_action_container *act_cnt = NULL;
14742         union {
14743                 struct mlx5_flow_dv_modify_hdr_resource res;
14744                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
14745                             sizeof(struct mlx5_modification_cmd) *
14746                             (MLX5_MAX_MODIFY_NUM + 1)];
14747         } mhdr_dummy;
14748
14749         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
14750         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
14751         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
14752         memset(&dev_flow, 0, sizeof(struct mlx5_flow));
14753         memset(&port_id_action, 0,
14754                 sizeof(struct mlx5_flow_dv_port_id_action_resource));
14755         dev_flow.handle = &dh;
14756         dev_flow.dv.port_id_action = &port_id_action;
14757         dev_flow.external = true;
14758         for (i = 0; i < RTE_COLORS; i++) {
14759                 if (i < MLX5_MTR_RTE_COLORS)
14760                         act_cnt = &mtr_policy->act_cnt[i];
14761                 for (act = actions[i];
14762                         act && act->type != RTE_FLOW_ACTION_TYPE_END;
14763                         act++) {
14764                         switch (act->type) {
14765                         case RTE_FLOW_ACTION_TYPE_MARK:
14766                         {
14767                                 uint32_t tag_be = mlx5_flow_mark_set
14768                                         (((const struct rte_flow_action_mark *)
14769                                         (act->conf))->id);
14770
14771                                 if (i >= MLX5_MTR_RTE_COLORS)
14772                                         return -rte_mtr_error_set(error,
14773                                           ENOTSUP,
14774                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14775                                           NULL,
14776                                           "cannot create policy "
14777                                           "mark action for this color");
14778                                 dev_flow.handle->mark = 1;
14779                                 if (flow_dv_tag_resource_register(dev, tag_be,
14780                                                   &dev_flow, &flow_err))
14781                                         return -rte_mtr_error_set(error,
14782                                         ENOTSUP,
14783                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14784                                         NULL,
14785                                         "cannot setup policy mark action");
14786                                 MLX5_ASSERT(dev_flow.dv.tag_resource);
14787                                 act_cnt->rix_mark =
14788                                         dev_flow.handle->dvh.rix_tag;
14789                                 action_flags |= MLX5_FLOW_ACTION_MARK;
14790                                 break;
14791                         }
14792                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
14793                         {
14794                                 struct mlx5_flow_dv_modify_hdr_resource
14795                                         *mhdr_res = &mhdr_dummy.res;
14796
14797                                 if (i >= MLX5_MTR_RTE_COLORS)
14798                                         return -rte_mtr_error_set(error,
14799                                           ENOTSUP,
14800                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14801                                           NULL,
14802                                           "cannot create policy "
14803                                           "set tag action for this color");
14804                                 memset(mhdr_res, 0, sizeof(*mhdr_res));
14805                                 mhdr_res->ft_type = transfer ?
14806                                         MLX5DV_FLOW_TABLE_TYPE_FDB :
14807                                         egress ?
14808                                         MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
14809                                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
14810                                 if (flow_dv_convert_action_set_tag
14811                                 (dev, mhdr_res,
14812                                 (const struct rte_flow_action_set_tag *)
14813                                 act->conf,  &flow_err))
14814                                         return -rte_mtr_error_set(error,
14815                                         ENOTSUP,
14816                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14817                                         NULL, "cannot convert policy "
14818                                         "set tag action");
14819                                 if (!mhdr_res->actions_num)
14820                                         return -rte_mtr_error_set(error,
14821                                         ENOTSUP,
14822                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14823                                         NULL, "cannot find policy "
14824                                         "set tag action");
14825                                 /* create modify action if needed. */
14826                                 dev_flow.dv.group = 1;
14827                                 if (flow_dv_modify_hdr_resource_register
14828                                         (dev, mhdr_res, &dev_flow, &flow_err))
14829                                         return -rte_mtr_error_set(error,
14830                                         ENOTSUP,
14831                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14832                                         NULL, "cannot register policy "
14833                                         "set tag action");
14834                                 act_cnt->modify_hdr =
14835                                 dev_flow.handle->dvh.modify_hdr;
14836                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
14837                                 break;
14838                         }
14839                         case RTE_FLOW_ACTION_TYPE_DROP:
14840                         {
14841                                 struct mlx5_flow_mtr_mng *mtrmng =
14842                                                 priv->sh->mtrmng;
14843                                 struct mlx5_flow_tbl_data_entry *tbl_data;
14844
14845                                 /*
14846                                  * Create the drop table with
14847                                  * METER DROP level.
14848                                  */
14849                                 if (!mtrmng->drop_tbl[domain]) {
14850                                         mtrmng->drop_tbl[domain] =
14851                                         flow_dv_tbl_resource_get(dev,
14852                                         MLX5_FLOW_TABLE_LEVEL_METER,
14853                                         egress, transfer, false, NULL, 0,
14854                                         0, MLX5_MTR_TABLE_ID_DROP, &flow_err);
14855                                         if (!mtrmng->drop_tbl[domain])
14856                                                 return -rte_mtr_error_set
14857                                         (error, ENOTSUP,
14858                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14859                                         NULL,
14860                                         "Failed to create meter drop table");
14861                                 }
14862                                 tbl_data = container_of
14863                                 (mtrmng->drop_tbl[domain],
14864                                 struct mlx5_flow_tbl_data_entry, tbl);
14865                                 if (i < MLX5_MTR_RTE_COLORS) {
14866                                         act_cnt->dr_jump_action[domain] =
14867                                                 tbl_data->jump.action;
14868                                         act_cnt->fate_action =
14869                                                 MLX5_FLOW_FATE_DROP;
14870                                 }
14871                                 if (i == RTE_COLOR_RED)
14872                                         mtr_policy->dr_drop_action[domain] =
14873                                                 tbl_data->jump.action;
14874                                 action_flags |= MLX5_FLOW_ACTION_DROP;
14875                                 break;
14876                         }
14877                         case RTE_FLOW_ACTION_TYPE_QUEUE:
14878                         {
14879                                 if (i >= MLX5_MTR_RTE_COLORS)
14880                                         return -rte_mtr_error_set(error,
14881                                         ENOTSUP,
14882                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14883                                         NULL, "cannot create policy "
14884                                         "fate queue for this color");
14885                                 act_cnt->queue =
14886                                 ((const struct rte_flow_action_queue *)
14887                                         (act->conf))->index;
14888                                 act_cnt->fate_action =
14889                                         MLX5_FLOW_FATE_QUEUE;
14890                                 dev_flow.handle->fate_action =
14891                                         MLX5_FLOW_FATE_QUEUE;
14892                                 mtr_policy->is_queue = 1;
14893                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
14894                                 break;
14895                         }
14896                         case RTE_FLOW_ACTION_TYPE_RSS:
14897                         {
14898                                 int rss_size;
14899
14900                                 if (i >= MLX5_MTR_RTE_COLORS)
14901                                         return -rte_mtr_error_set(error,
14902                                           ENOTSUP,
14903                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14904                                           NULL,
14905                                           "cannot create policy "
14906                                           "rss action for this color");
14907                                 /*
14908                                  * Save RSS conf into policy struct
14909                                  * for translate stage.
14910                                  */
14911                                 rss_size = (int)rte_flow_conv
14912                                         (RTE_FLOW_CONV_OP_ACTION,
14913                                         NULL, 0, act, &flow_err);
14914                                 if (rss_size <= 0)
14915                                         return -rte_mtr_error_set(error,
14916                                           ENOTSUP,
14917                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14918                                           NULL, "Get the wrong "
14919                                           "rss action struct size");
14920                                 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO,
14921                                                 rss_size, 0, SOCKET_ID_ANY);
14922                                 if (!act_cnt->rss)
14923                                         return -rte_mtr_error_set(error,
14924                                           ENOTSUP,
14925                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14926                                           NULL,
14927                                           "Fail to malloc rss action memory");
14928                                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION,
14929                                         act_cnt->rss, rss_size,
14930                                         act, &flow_err);
14931                                 if (ret < 0)
14932                                         return -rte_mtr_error_set(error,
14933                                           ENOTSUP,
14934                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14935                                           NULL, "Fail to save "
14936                                           "rss action into policy struct");
14937                                 act_cnt->fate_action =
14938                                         MLX5_FLOW_FATE_SHARED_RSS;
14939                                 action_flags |= MLX5_FLOW_ACTION_RSS;
14940                                 break;
14941                         }
14942                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
14943                         {
14944                                 struct mlx5_flow_dv_port_id_action_resource
14945                                         port_id_resource;
14946                                 uint32_t port_id = 0;
14947
14948                                 if (i >= MLX5_MTR_RTE_COLORS)
14949                                         return -rte_mtr_error_set(error,
14950                                         ENOTSUP,
14951                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14952                                         NULL, "cannot create policy "
14953                                         "port action for this color");
14954                                 memset(&port_id_resource, 0,
14955                                         sizeof(port_id_resource));
14956                                 if (flow_dv_translate_action_port_id(dev, act,
14957                                                 &port_id, &flow_err))
14958                                         return -rte_mtr_error_set(error,
14959                                         ENOTSUP,
14960                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14961                                         NULL, "cannot translate "
14962                                         "policy port action");
14963                                 port_id_resource.port_id = port_id;
14964                                 if (flow_dv_port_id_action_resource_register
14965                                         (dev, &port_id_resource,
14966                                         &dev_flow, &flow_err))
14967                                         return -rte_mtr_error_set(error,
14968                                         ENOTSUP,
14969                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
14970                                         NULL, "cannot setup "
14971                                         "policy port action");
14972                                 act_cnt->rix_port_id_action =
14973                                         dev_flow.handle->rix_port_id_action;
14974                                 act_cnt->fate_action =
14975                                         MLX5_FLOW_FATE_PORT_ID;
14976                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
14977                                 break;
14978                         }
14979                         case RTE_FLOW_ACTION_TYPE_JUMP:
14980                         {
14981                                 uint32_t jump_group = 0;
14982                                 uint32_t table = 0;
14983                                 struct mlx5_flow_tbl_data_entry *tbl_data;
14984                                 struct flow_grp_info grp_info = {
14985                                         .external = !!dev_flow.external,
14986                                         .transfer = !!transfer,
14987                                         .fdb_def_rule = !!priv->fdb_def_rule,
14988                                         .std_tbl_fix = 0,
14989                                         .skip_scale = dev_flow.skip_scale &
14990                                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
14991                                 };
14992                                 struct mlx5_flow_meter_sub_policy *sub_policy =
14993                                 mtr_policy->sub_policys[domain][0];
14994
14995                                 if (i >= MLX5_MTR_RTE_COLORS)
14996                                         return -rte_mtr_error_set(error,
14997                                           ENOTSUP,
14998                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
14999                                           NULL,
15000                                           "cannot create policy "
15001                                           "jump action for this color");
15002                                 jump_group =
15003                                 ((const struct rte_flow_action_jump *)
15004                                                         act->conf)->group;
15005                                 if (mlx5_flow_group_to_table(dev, NULL,
15006                                                        jump_group,
15007                                                        &table,
15008                                                        &grp_info, &flow_err))
15009                                         return -rte_mtr_error_set(error,
15010                                         ENOTSUP,
15011                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15012                                         NULL, "cannot setup "
15013                                         "policy jump action");
15014                                 sub_policy->jump_tbl[i] =
15015                                 flow_dv_tbl_resource_get(dev,
15016                                         table, egress,
15017                                         transfer,
15018                                         !!dev_flow.external,
15019                                         NULL, jump_group, 0,
15020                                         0, &flow_err);
15021                                 if
15022                                 (!sub_policy->jump_tbl[i])
15023                                         return  -rte_mtr_error_set(error,
15024                                         ENOTSUP,
15025                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15026                                         NULL, "cannot create jump action.");
15027                                 tbl_data = container_of
15028                                 (sub_policy->jump_tbl[i],
15029                                 struct mlx5_flow_tbl_data_entry, tbl);
15030                                 act_cnt->dr_jump_action[domain] =
15031                                         tbl_data->jump.action;
15032                                 act_cnt->fate_action =
15033                                         MLX5_FLOW_FATE_JUMP;
15034                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
15035                                 break;
15036                         }
15037                         case RTE_FLOW_ACTION_TYPE_METER:
15038                         {
15039                                 const struct rte_flow_action_meter *mtr;
15040                                 struct mlx5_flow_meter_info *next_fm;
15041                                 struct mlx5_flow_meter_policy *next_policy;
15042                                 uint32_t next_mtr_idx = 0;
15043
15044                                 mtr = act->conf;
15045                                 next_fm = mlx5_flow_meter_find(priv,
15046                                                         mtr->mtr_id,
15047                                                         &next_mtr_idx);
15048                                 if (!next_fm)
15049                                         return -rte_mtr_error_set(error, EINVAL,
15050                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15051                                                 "Fail to find next meter.");
15052                                 if (next_fm->def_policy)
15053                                         return -rte_mtr_error_set(error, EINVAL,
15054                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15055                                 "Hierarchy only supports termination meter.");
15056                                 next_policy = mlx5_flow_meter_policy_find(dev,
15057                                                 next_fm->policy_id, NULL);
15058                                 MLX5_ASSERT(next_policy);
15059                                 act_cnt->fate_action = MLX5_FLOW_FATE_MTR;
15060                                 act_cnt->next_mtr_id = next_fm->meter_id;
15061                                 act_cnt->next_sub_policy = NULL;
15062                                 mtr_policy->is_hierarchy = 1;
15063                                 mtr_policy->dev = next_policy->dev;
15064                                 action_flags |=
15065                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
15066                                 break;
15067                         }
15068                         default:
15069                                 return -rte_mtr_error_set(error, ENOTSUP,
15070                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15071                                           NULL, "action type not supported");
15072                         }
15073                 }
15074         }
15075         return 0;
15076 }
15077
15078 /**
15079  * Create policy action per domain, lock free,
15080  * (mutex should be acquired by caller).
15081  * Dispatcher for action type specific call.
15082  *
15083  * @param[in] dev
15084  *   Pointer to the Ethernet device structure.
15085  * @param[in] mtr_policy
15086  *   Meter policy struct.
15087  * @param[in] action
15088  *   Action specification used to create meter actions.
15089  * @param[out] error
15090  *   Perform verbose error reporting if not NULL. Initialized in case of
15091  *   error only.
15092  *
15093  * @return
15094  *   0 on success, otherwise negative errno value.
15095  */
15096 static int
15097 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev,
15098                       struct mlx5_flow_meter_policy *mtr_policy,
15099                       const struct rte_flow_action *actions[RTE_COLORS],
15100                       struct rte_mtr_error *error)
15101 {
15102         int ret, i;
15103         uint16_t sub_policy_num;
15104
15105         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15106                 sub_policy_num = (mtr_policy->sub_policy_num >>
15107                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15108                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15109                 if (sub_policy_num) {
15110                         ret = __flow_dv_create_domain_policy_acts(dev,
15111                                 mtr_policy, actions,
15112                                 (enum mlx5_meter_domain)i, error);
15113                         if (ret)
15114                                 return ret;
15115                 }
15116         }
15117         return 0;
15118 }
15119
15120 /**
15121  * Query a DV flow rule for its statistics via DevX.
15122  *
15123  * @param[in] dev
15124  *   Pointer to Ethernet device.
15125  * @param[in] cnt_idx
15126  *   Index to the flow counter.
15127  * @param[out] data
15128  *   Data retrieved by the query.
15129  * @param[out] error
15130  *   Perform verbose error reporting if not NULL.
15131  *
15132  * @return
15133  *   0 on success, a negative errno value otherwise and rte_errno is set.
15134  */
15135 static int
15136 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data,
15137                     struct rte_flow_error *error)
15138 {
15139         struct mlx5_priv *priv = dev->data->dev_private;
15140         struct rte_flow_query_count *qc = data;
15141
15142         if (!priv->config.devx)
15143                 return rte_flow_error_set(error, ENOTSUP,
15144                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15145                                           NULL,
15146                                           "counters are not supported");
15147         if (cnt_idx) {
15148                 uint64_t pkts, bytes;
15149                 struct mlx5_flow_counter *cnt;
15150                 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes);
15151
15152                 if (err)
15153                         return rte_flow_error_set(error, -err,
15154                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15155                                         NULL, "cannot read counters");
15156                 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL);
15157                 qc->hits_set = 1;
15158                 qc->bytes_set = 1;
15159                 qc->hits = pkts - cnt->hits;
15160                 qc->bytes = bytes - cnt->bytes;
15161                 if (qc->reset) {
15162                         cnt->hits = pkts;
15163                         cnt->bytes = bytes;
15164                 }
15165                 return 0;
15166         }
15167         return rte_flow_error_set(error, EINVAL,
15168                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15169                                   NULL,
15170                                   "counters are not available");
15171 }
15172
15173 static int
15174 flow_dv_action_query(struct rte_eth_dev *dev,
15175                      const struct rte_flow_action_handle *handle, void *data,
15176                      struct rte_flow_error *error)
15177 {
15178         struct mlx5_age_param *age_param;
15179         struct rte_flow_query_age *resp;
15180         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15181         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15182         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15183         struct mlx5_priv *priv = dev->data->dev_private;
15184         struct mlx5_aso_ct_action *ct;
15185         uint16_t owner;
15186         uint32_t dev_idx;
15187
15188         switch (type) {
15189         case MLX5_INDIRECT_ACTION_TYPE_AGE:
15190                 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params;
15191                 resp = data;
15192                 resp->aged = __atomic_load_n(&age_param->state,
15193                                               __ATOMIC_RELAXED) == AGE_TMOUT ?
15194                                                                           1 : 0;
15195                 resp->sec_since_last_hit_valid = !resp->aged;
15196                 if (resp->sec_since_last_hit_valid)
15197                         resp->sec_since_last_hit = __atomic_load_n
15198                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15199                 return 0;
15200         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
15201                 return flow_dv_query_count(dev, idx, data, error);
15202         case MLX5_INDIRECT_ACTION_TYPE_CT:
15203                 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15204                 if (owner != PORT_ID(priv))
15205                         return rte_flow_error_set(error, EACCES,
15206                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15207                                         NULL,
15208                                         "CT object owned by another port");
15209                 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15210                 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15211                 MLX5_ASSERT(ct);
15212                 if (!ct->refcnt)
15213                         return rte_flow_error_set(error, EFAULT,
15214                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15215                                         NULL,
15216                                         "CT object is inactive");
15217                 ((struct rte_flow_action_conntrack *)data)->peer_port =
15218                                                         ct->peer;
15219                 ((struct rte_flow_action_conntrack *)data)->is_original_dir =
15220                                                         ct->is_original;
15221                 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data))
15222                         return rte_flow_error_set(error, EIO,
15223                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15224                                         NULL,
15225                                         "Failed to query CT context");
15226                 return 0;
15227         default:
15228                 return rte_flow_error_set(error, ENOTSUP,
15229                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15230                                           "action type query not supported");
15231         }
15232 }
15233
15234 /**
15235  * Query a flow rule AGE action for aging information.
15236  *
15237  * @param[in] dev
15238  *   Pointer to Ethernet device.
15239  * @param[in] flow
15240  *   Pointer to the sub flow.
15241  * @param[out] data
15242  *   data retrieved by the query.
15243  * @param[out] error
15244  *   Perform verbose error reporting if not NULL.
15245  *
15246  * @return
15247  *   0 on success, a negative errno value otherwise and rte_errno is set.
15248  */
15249 static int
15250 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
15251                   void *data, struct rte_flow_error *error)
15252 {
15253         struct rte_flow_query_age *resp = data;
15254         struct mlx5_age_param *age_param;
15255
15256         if (flow->age) {
15257                 struct mlx5_aso_age_action *act =
15258                                      flow_aso_age_get_by_idx(dev, flow->age);
15259
15260                 age_param = &act->age_params;
15261         } else if (flow->counter) {
15262                 age_param = flow_dv_counter_idx_get_age(dev, flow->counter);
15263
15264                 if (!age_param || !age_param->timeout)
15265                         return rte_flow_error_set
15266                                         (error, EINVAL,
15267                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15268                                          NULL, "cannot read age data");
15269         } else {
15270                 return rte_flow_error_set(error, EINVAL,
15271                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15272                                           NULL, "age data not available");
15273         }
15274         resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
15275                                      AGE_TMOUT ? 1 : 0;
15276         resp->sec_since_last_hit_valid = !resp->aged;
15277         if (resp->sec_since_last_hit_valid)
15278                 resp->sec_since_last_hit = __atomic_load_n
15279                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15280         return 0;
15281 }
15282
15283 /**
15284  * Query a flow.
15285  *
15286  * @see rte_flow_query()
15287  * @see rte_flow_ops
15288  */
15289 static int
15290 flow_dv_query(struct rte_eth_dev *dev,
15291               struct rte_flow *flow __rte_unused,
15292               const struct rte_flow_action *actions __rte_unused,
15293               void *data __rte_unused,
15294               struct rte_flow_error *error __rte_unused)
15295 {
15296         int ret = -EINVAL;
15297
15298         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
15299                 switch (actions->type) {
15300                 case RTE_FLOW_ACTION_TYPE_VOID:
15301                         break;
15302                 case RTE_FLOW_ACTION_TYPE_COUNT:
15303                         ret = flow_dv_query_count(dev, flow->counter, data,
15304                                                   error);
15305                         break;
15306                 case RTE_FLOW_ACTION_TYPE_AGE:
15307                         ret = flow_dv_query_age(dev, flow, data, error);
15308                         break;
15309                 default:
15310                         return rte_flow_error_set(error, ENOTSUP,
15311                                                   RTE_FLOW_ERROR_TYPE_ACTION,
15312                                                   actions,
15313                                                   "action not supported");
15314                 }
15315         }
15316         return ret;
15317 }
15318
15319 /**
15320  * Destroy the meter table set.
15321  * Lock free, (mutex should be acquired by caller).
15322  *
15323  * @param[in] dev
15324  *   Pointer to Ethernet device.
15325  * @param[in] fm
15326  *   Meter information table.
15327  */
15328 static void
15329 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
15330                         struct mlx5_flow_meter_info *fm)
15331 {
15332         struct mlx5_priv *priv = dev->data->dev_private;
15333         int i;
15334
15335         if (!fm || !priv->config.dv_flow_en)
15336                 return;
15337         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15338                 if (fm->drop_rule[i]) {
15339                         claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
15340                         fm->drop_rule[i] = NULL;
15341                 }
15342         }
15343 }
15344
15345 static void
15346 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
15347 {
15348         struct mlx5_priv *priv = dev->data->dev_private;
15349         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15350         struct mlx5_flow_tbl_data_entry *tbl;
15351         int i, j;
15352
15353         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15354                 if (mtrmng->def_rule[i]) {
15355                         claim_zero(mlx5_flow_os_destroy_flow
15356                                         (mtrmng->def_rule[i]));
15357                         mtrmng->def_rule[i] = NULL;
15358                 }
15359                 if (mtrmng->def_matcher[i]) {
15360                         tbl = container_of(mtrmng->def_matcher[i]->tbl,
15361                                 struct mlx5_flow_tbl_data_entry, tbl);
15362                         mlx5_cache_unregister(&tbl->matchers,
15363                                       &mtrmng->def_matcher[i]->entry);
15364                         mtrmng->def_matcher[i] = NULL;
15365                 }
15366                 for (j = 0; j < MLX5_REG_BITS; j++) {
15367                         if (mtrmng->drop_matcher[i][j]) {
15368                                 tbl =
15369                                 container_of(mtrmng->drop_matcher[i][j]->tbl,
15370                                              struct mlx5_flow_tbl_data_entry,
15371                                              tbl);
15372                                 mlx5_cache_unregister(&tbl->matchers,
15373                                         &mtrmng->drop_matcher[i][j]->entry);
15374                                 mtrmng->drop_matcher[i][j] = NULL;
15375                         }
15376                 }
15377                 if (mtrmng->drop_tbl[i]) {
15378                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15379                                 mtrmng->drop_tbl[i]);
15380                         mtrmng->drop_tbl[i] = NULL;
15381                 }
15382         }
15383 }
15384
15385 /* Number of meter flow actions, count and jump or count and drop. */
15386 #define METER_ACTIONS 2
15387
15388 static void
15389 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
15390                               enum mlx5_meter_domain domain)
15391 {
15392         struct mlx5_priv *priv = dev->data->dev_private;
15393         struct mlx5_flow_meter_def_policy *def_policy =
15394                         priv->sh->mtrmng->def_policy[domain];
15395
15396         __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
15397         mlx5_free(def_policy);
15398         priv->sh->mtrmng->def_policy[domain] = NULL;
15399 }
15400
15401 /**
15402  * Destroy the default policy table set.
15403  *
15404  * @param[in] dev
15405  *   Pointer to Ethernet device.
15406  */
15407 static void
15408 flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
15409 {
15410         struct mlx5_priv *priv = dev->data->dev_private;
15411         int i;
15412
15413         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
15414                 if (priv->sh->mtrmng->def_policy[i])
15415                         __flow_dv_destroy_domain_def_policy(dev,
15416                                         (enum mlx5_meter_domain)i);
15417         priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
15418 }
15419
15420 static int
15421 __flow_dv_create_policy_flow(struct rte_eth_dev *dev,
15422                         uint32_t color_reg_c_idx,
15423                         enum rte_color color, void *matcher_object,
15424                         int actions_n, void *actions,
15425                         bool match_src_port, void **rule,
15426                         const struct rte_flow_attr *attr)
15427 {
15428         int ret;
15429         struct mlx5_flow_dv_match_params value = {
15430                 .size = sizeof(value.buf) -
15431                         MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15432         };
15433         struct mlx5_flow_dv_match_params matcher = {
15434                 .size = sizeof(matcher.buf) -
15435                         MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15436         };
15437         struct mlx5_priv *priv = dev->data->dev_private;
15438
15439         if (match_src_port && (priv->representor || priv->master)) {
15440                 if (flow_dv_translate_item_port_id(dev, matcher.buf,
15441                                                    value.buf, NULL, attr)) {
15442                         DRV_LOG(ERR,
15443                         "Failed to create meter policy flow with port.");
15444                         return -1;
15445                 }
15446         }
15447         flow_dv_match_meta_reg(matcher.buf, value.buf,
15448                                 (enum modify_reg)color_reg_c_idx,
15449                                 rte_col_2_mlx5_col(color),
15450                                 UINT32_MAX);
15451         ret = mlx5_flow_os_create_flow(matcher_object,
15452                         (void *)&value, actions_n, actions, rule);
15453         if (ret) {
15454                 DRV_LOG(ERR, "Failed to create meter policy flow.");
15455                 return -1;
15456         }
15457         return 0;
15458 }
15459
15460 static int
15461 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
15462                         uint32_t color_reg_c_idx,
15463                         uint16_t priority,
15464                         struct mlx5_flow_meter_sub_policy *sub_policy,
15465                         const struct rte_flow_attr *attr,
15466                         bool match_src_port,
15467                         struct rte_flow_error *error)
15468 {
15469         struct mlx5_cache_entry *entry;
15470         struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
15471         struct mlx5_flow_dv_matcher matcher = {
15472                 .mask = {
15473                         .size = sizeof(matcher.mask.buf) -
15474                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15475                 },
15476                 .tbl = tbl_rsc,
15477         };
15478         struct mlx5_flow_dv_match_params value = {
15479                 .size = sizeof(value.buf) -
15480                         MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15481         };
15482         struct mlx5_flow_cb_ctx ctx = {
15483                 .error = error,
15484                 .data = &matcher,
15485         };
15486         struct mlx5_flow_tbl_data_entry *tbl_data;
15487         struct mlx5_priv *priv = dev->data->dev_private;
15488         uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
15489
15490         if (match_src_port && (priv->representor || priv->master)) {
15491                 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
15492                                                    value.buf, NULL, attr)) {
15493                         DRV_LOG(ERR,
15494                         "Failed to register meter drop matcher with port.");
15495                         return -1;
15496                 }
15497         }
15498         tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
15499         if (priority < RTE_COLOR_RED)
15500                 flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
15501                         (enum modify_reg)color_reg_c_idx, 0, color_mask);
15502         matcher.priority = priority;
15503         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
15504                                         matcher.mask.size);
15505         entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
15506         if (!entry) {
15507                 DRV_LOG(ERR, "Failed to register meter drop matcher.");
15508                 return -1;
15509         }
15510         sub_policy->color_matcher[priority] =
15511                 container_of(entry, struct mlx5_flow_dv_matcher, entry);
15512         return 0;
15513 }
15514
15515 /**
15516  * Create the policy rules per domain.
15517  *
15518  * @param[in] dev
15519  *   Pointer to Ethernet device.
15520  * @param[in] sub_policy
15521  *    Pointer to sub policy table..
15522  * @param[in] egress
15523  *   Direction of the table.
15524  * @param[in] transfer
15525  *   E-Switch or NIC flow.
15526  * @param[in] acts
15527  *   Pointer to policy action list per color.
15528  *
15529  * @return
15530  *   0 on success, -1 otherwise.
15531  */
15532 static int
15533 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
15534                 struct mlx5_flow_meter_sub_policy *sub_policy,
15535                 uint8_t egress, uint8_t transfer, bool match_src_port,
15536                 struct mlx5_meter_policy_acts acts[RTE_COLORS])
15537 {
15538         struct rte_flow_error flow_err;
15539         uint32_t color_reg_c_idx;
15540         struct rte_flow_attr attr = {
15541                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
15542                 .priority = 0,
15543                 .ingress = 0,
15544                 .egress = !!egress,
15545                 .transfer = !!transfer,
15546                 .reserved = 0,
15547         };
15548         int i;
15549         int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
15550
15551         if (ret < 0)
15552                 return -1;
15553         /* Create policy table with POLICY level. */
15554         if (!sub_policy->tbl_rsc)
15555                 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
15556                                 MLX5_FLOW_TABLE_LEVEL_POLICY,
15557                                 egress, transfer, false, NULL, 0, 0,
15558                                 sub_policy->idx, &flow_err);
15559         if (!sub_policy->tbl_rsc) {
15560                 DRV_LOG(ERR,
15561                         "Failed to create meter sub policy table.");
15562                 return -1;
15563         }
15564         /* Prepare matchers. */
15565         color_reg_c_idx = ret;
15566         for (i = 0; i < RTE_COLORS; i++) {
15567                 if (i == RTE_COLOR_YELLOW || !acts[i].actions_n)
15568                         continue;
15569                 attr.priority = i;
15570                 if (!sub_policy->color_matcher[i]) {
15571                         /* Create matchers for Color. */
15572                         if (__flow_dv_create_policy_matcher(dev,
15573                                 color_reg_c_idx, i, sub_policy,
15574                                 &attr, match_src_port, &flow_err))
15575                                 return -1;
15576                 }
15577                 /* Create flow, matching color. */
15578                 if (acts[i].actions_n)
15579                         if (__flow_dv_create_policy_flow(dev,
15580                                 color_reg_c_idx, (enum rte_color)i,
15581                                 sub_policy->color_matcher[i]->matcher_object,
15582                                 acts[i].actions_n,
15583                                 acts[i].dv_actions,
15584                                 match_src_port,
15585                                 &sub_policy->color_rule[i],
15586                                 &attr))
15587                                 return -1;
15588         }
15589         return 0;
15590 }
15591
15592 static int
15593 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
15594                         struct mlx5_flow_meter_policy *mtr_policy,
15595                         struct mlx5_flow_meter_sub_policy *sub_policy,
15596                         uint32_t domain)
15597 {
15598         struct mlx5_priv *priv = dev->data->dev_private;
15599         struct mlx5_meter_policy_acts acts[RTE_COLORS];
15600         struct mlx5_flow_dv_tag_resource *tag;
15601         struct mlx5_flow_dv_port_id_action_resource *port_action;
15602         struct mlx5_hrxq *hrxq;
15603         struct mlx5_flow_meter_info *next_fm = NULL;
15604         struct mlx5_flow_meter_policy *next_policy;
15605         struct mlx5_flow_meter_sub_policy *next_sub_policy;
15606         struct mlx5_flow_tbl_data_entry *tbl_data;
15607         struct rte_flow_error error;
15608         uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15609         uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15610         bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX);
15611         bool match_src_port = false;
15612         int i;
15613
15614         for (i = 0; i < RTE_COLORS; i++) {
15615                 acts[i].actions_n = 0;
15616                 if (i == RTE_COLOR_YELLOW)
15617                         continue;
15618                 if (i == RTE_COLOR_RED) {
15619                         /* Only support drop on red. */
15620                         acts[i].dv_actions[0] =
15621                         mtr_policy->dr_drop_action[domain];
15622                         acts[i].actions_n = 1;
15623                         continue;
15624                 }
15625                 if (mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
15626                         struct rte_flow_attr attr = {
15627                                 .transfer = transfer
15628                         };
15629
15630                         next_fm = mlx5_flow_meter_find(priv,
15631                                         mtr_policy->act_cnt[i].next_mtr_id,
15632                                         NULL);
15633                         if (!next_fm) {
15634                                 DRV_LOG(ERR,
15635                                         "Failed to get next hierarchy meter.");
15636                                 goto err_exit;
15637                         }
15638                         if (mlx5_flow_meter_attach(priv, next_fm,
15639                                                    &attr, &error)) {
15640                                 DRV_LOG(ERR, "%s", error.message);
15641                                 next_fm = NULL;
15642                                 goto err_exit;
15643                         }
15644                         /* Meter action must be the first for TX. */
15645                         if (mtr_first) {
15646                                 acts[i].dv_actions[acts[i].actions_n] =
15647                                         next_fm->meter_action;
15648                                 acts[i].actions_n++;
15649                         }
15650                 }
15651                 if (mtr_policy->act_cnt[i].rix_mark) {
15652                         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
15653                                         mtr_policy->act_cnt[i].rix_mark);
15654                         if (!tag) {
15655                                 DRV_LOG(ERR, "Failed to find "
15656                                 "mark action for policy.");
15657                                 goto err_exit;
15658                         }
15659                         acts[i].dv_actions[acts[i].actions_n] =
15660                                                 tag->action;
15661                         acts[i].actions_n++;
15662                 }
15663                 if (mtr_policy->act_cnt[i].modify_hdr) {
15664                         acts[i].dv_actions[acts[i].actions_n] =
15665                         mtr_policy->act_cnt[i].modify_hdr->action;
15666                         acts[i].actions_n++;
15667                 }
15668                 if (mtr_policy->act_cnt[i].fate_action) {
15669                         switch (mtr_policy->act_cnt[i].fate_action) {
15670                         case MLX5_FLOW_FATE_PORT_ID:
15671                                 port_action = mlx5_ipool_get
15672                                         (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
15673                                 mtr_policy->act_cnt[i].rix_port_id_action);
15674                                 if (!port_action) {
15675                                         DRV_LOG(ERR, "Failed to find "
15676                                                 "port action for policy.");
15677                                         goto err_exit;
15678                                 }
15679                                 acts[i].dv_actions[acts[i].actions_n] =
15680                                 port_action->action;
15681                                 acts[i].actions_n++;
15682                                 mtr_policy->dev = dev;
15683                                 match_src_port = true;
15684                                 break;
15685                         case MLX5_FLOW_FATE_DROP:
15686                         case MLX5_FLOW_FATE_JUMP:
15687                                 acts[i].dv_actions[acts[i].actions_n] =
15688                                 mtr_policy->act_cnt[i].dr_jump_action[domain];
15689                                 acts[i].actions_n++;
15690                                 break;
15691                         case MLX5_FLOW_FATE_SHARED_RSS:
15692                         case MLX5_FLOW_FATE_QUEUE:
15693                                 hrxq = mlx5_ipool_get
15694                                 (priv->sh->ipool[MLX5_IPOOL_HRXQ],
15695                                 sub_policy->rix_hrxq[i]);
15696                                 if (!hrxq) {
15697                                         DRV_LOG(ERR, "Failed to find "
15698                                                 "queue action for policy.");
15699                                         goto err_exit;
15700                                 }
15701                                 acts[i].dv_actions[acts[i].actions_n] =
15702                                 hrxq->action;
15703                                 acts[i].actions_n++;
15704                                 break;
15705                         case MLX5_FLOW_FATE_MTR:
15706                                 if (!next_fm) {
15707                                         DRV_LOG(ERR,
15708                                                 "No next hierarchy meter.");
15709                                         goto err_exit;
15710                                 }
15711                                 if (!mtr_first) {
15712                                         acts[i].dv_actions[acts[i].actions_n] =
15713                                                         next_fm->meter_action;
15714                                         acts[i].actions_n++;
15715                                 }
15716                                 if (mtr_policy->act_cnt[i].next_sub_policy) {
15717                                         next_sub_policy =
15718                                         mtr_policy->act_cnt[i].next_sub_policy;
15719                                 } else {
15720                                         next_policy =
15721                                                 mlx5_flow_meter_policy_find(dev,
15722                                                 next_fm->policy_id, NULL);
15723                                         MLX5_ASSERT(next_policy);
15724                                         next_sub_policy =
15725                                         next_policy->sub_policys[domain][0];
15726                                 }
15727                                 tbl_data =
15728                                         container_of(next_sub_policy->tbl_rsc,
15729                                         struct mlx5_flow_tbl_data_entry, tbl);
15730                                 acts[i].dv_actions[acts[i].actions_n++] =
15731                                                         tbl_data->jump.action;
15732                                 if (mtr_policy->act_cnt[i].modify_hdr)
15733                                         match_src_port = !!transfer;
15734                                 break;
15735                         default:
15736                                 /*Queue action do nothing*/
15737                                 break;
15738                         }
15739                 }
15740         }
15741         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15742         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15743         if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
15744                                 egress, transfer, match_src_port, acts)) {
15745                 DRV_LOG(ERR,
15746                 "Failed to create policy rules per domain.");
15747                 goto err_exit;
15748         }
15749         return 0;
15750 err_exit:
15751         if (next_fm)
15752                 mlx5_flow_meter_detach(priv, next_fm);
15753         return -1;
15754 }
15755
15756 /**
15757  * Create the policy rules.
15758  *
15759  * @param[in] dev
15760  *   Pointer to Ethernet device.
15761  * @param[in,out] mtr_policy
15762  *   Pointer to meter policy table.
15763  *
15764  * @return
15765  *   0 on success, -1 otherwise.
15766  */
15767 static int
15768 flow_dv_create_policy_rules(struct rte_eth_dev *dev,
15769                              struct mlx5_flow_meter_policy *mtr_policy)
15770 {
15771         int i;
15772         uint16_t sub_policy_num;
15773
15774         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15775                 sub_policy_num = (mtr_policy->sub_policy_num >>
15776                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15777                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15778                 if (!sub_policy_num)
15779                         continue;
15780                 /* Prepare actions list and create policy rules. */
15781                 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
15782                         mtr_policy->sub_policys[i][0], i)) {
15783                         DRV_LOG(ERR,
15784                         "Failed to create policy action list per domain.");
15785                         return -1;
15786                 }
15787         }
15788         return 0;
15789 }
15790
15791 static int
15792 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
15793 {
15794         struct mlx5_priv *priv = dev->data->dev_private;
15795         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15796         struct mlx5_flow_meter_def_policy *def_policy;
15797         struct mlx5_flow_tbl_resource *jump_tbl;
15798         struct mlx5_flow_tbl_data_entry *tbl_data;
15799         uint8_t egress, transfer;
15800         struct rte_flow_error error;
15801         struct mlx5_meter_policy_acts acts[RTE_COLORS];
15802         int ret;
15803
15804         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15805         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15806         def_policy = mtrmng->def_policy[domain];
15807         if (!def_policy) {
15808                 def_policy = mlx5_malloc(MLX5_MEM_ZERO,
15809                         sizeof(struct mlx5_flow_meter_def_policy),
15810                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
15811                 if (!def_policy) {
15812                         DRV_LOG(ERR, "Failed to alloc "
15813                                         "default policy table.");
15814                         goto def_policy_error;
15815                 }
15816                 mtrmng->def_policy[domain] = def_policy;
15817                 /* Create the meter suffix table with SUFFIX level. */
15818                 jump_tbl = flow_dv_tbl_resource_get(dev,
15819                                 MLX5_FLOW_TABLE_LEVEL_METER,
15820                                 egress, transfer, false, NULL, 0,
15821                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
15822                 if (!jump_tbl) {
15823                         DRV_LOG(ERR,
15824                                 "Failed to create meter suffix table.");
15825                         goto def_policy_error;
15826                 }
15827                 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
15828                 tbl_data = container_of(jump_tbl,
15829                                 struct mlx5_flow_tbl_data_entry, tbl);
15830                 def_policy->dr_jump_action[RTE_COLOR_GREEN] =
15831                                                 tbl_data->jump.action;
15832                 acts[RTE_COLOR_GREEN].dv_actions[0] =
15833                                                 tbl_data->jump.action;
15834                 acts[RTE_COLOR_GREEN].actions_n = 1;
15835                 /* Create jump action to the drop table. */
15836                 if (!mtrmng->drop_tbl[domain]) {
15837                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
15838                                 (dev, MLX5_FLOW_TABLE_LEVEL_METER,
15839                                 egress, transfer, false, NULL, 0,
15840                                 0, MLX5_MTR_TABLE_ID_DROP, &error);
15841                         if (!mtrmng->drop_tbl[domain]) {
15842                                 DRV_LOG(ERR, "Failed to create "
15843                                 "meter drop table for default policy.");
15844                                 goto def_policy_error;
15845                         }
15846                 }
15847                 tbl_data = container_of(mtrmng->drop_tbl[domain],
15848                                 struct mlx5_flow_tbl_data_entry, tbl);
15849                 def_policy->dr_jump_action[RTE_COLOR_RED] =
15850                                                 tbl_data->jump.action;
15851                 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
15852                 acts[RTE_COLOR_RED].actions_n = 1;
15853                 /* Create default policy rules. */
15854                 ret = __flow_dv_create_domain_policy_rules(dev,
15855                                         &def_policy->sub_policy,
15856                                         egress, transfer, false, acts);
15857                 if (ret) {
15858                         DRV_LOG(ERR, "Failed to create "
15859                                 "default policy rules.");
15860                                 goto def_policy_error;
15861                 }
15862         }
15863         return 0;
15864 def_policy_error:
15865         __flow_dv_destroy_domain_def_policy(dev,
15866                         (enum mlx5_meter_domain)domain);
15867         return -1;
15868 }
15869
15870 /**
15871  * Create the default policy table set.
15872  *
15873  * @param[in] dev
15874  *   Pointer to Ethernet device.
15875  * @return
15876  *   0 on success, -1 otherwise.
15877  */
15878 static int
15879 flow_dv_create_def_policy(struct rte_eth_dev *dev)
15880 {
15881         struct mlx5_priv *priv = dev->data->dev_private;
15882         int i;
15883
15884         /* Non-termination policy table. */
15885         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15886                 if (!priv->config.dv_esw_en && i == MLX5_MTR_DOMAIN_TRANSFER)
15887                         continue;
15888                 if (__flow_dv_create_domain_def_policy(dev, i)) {
15889                         DRV_LOG(ERR,
15890                         "Failed to create default policy");
15891                         return -1;
15892                 }
15893         }
15894         return 0;
15895 }
15896
15897 /**
15898  * Create the needed meter tables.
15899  * Lock free, (mutex should be acquired by caller).
15900  *
15901  * @param[in] dev
15902  *   Pointer to Ethernet device.
15903  * @param[in] fm
15904  *   Meter information table.
15905  * @param[in] mtr_idx
15906  *   Meter index.
15907  * @param[in] domain_bitmap
15908  *   Domain bitmap.
15909  * @return
15910  *   0 on success, -1 otherwise.
15911  */
15912 static int
15913 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
15914                         struct mlx5_flow_meter_info *fm,
15915                         uint32_t mtr_idx,
15916                         uint8_t domain_bitmap)
15917 {
15918         struct mlx5_priv *priv = dev->data->dev_private;
15919         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15920         struct rte_flow_error error;
15921         struct mlx5_flow_tbl_data_entry *tbl_data;
15922         uint8_t egress, transfer;
15923         void *actions[METER_ACTIONS];
15924         int domain, ret, i;
15925         struct mlx5_flow_counter *cnt;
15926         struct mlx5_flow_dv_match_params value = {
15927                 .size = sizeof(value.buf) -
15928                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15929         };
15930         struct mlx5_flow_dv_match_params matcher_para = {
15931                 .size = sizeof(matcher_para.buf) -
15932                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15933         };
15934         int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
15935                                                      0, &error);
15936         uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
15937         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
15938         struct mlx5_cache_entry *entry;
15939         struct mlx5_flow_dv_matcher matcher = {
15940                 .mask = {
15941                         .size = sizeof(matcher.mask.buf) -
15942                         MLX5_ST_SZ_BYTES(fte_match_set_misc4),
15943                 },
15944         };
15945         struct mlx5_flow_dv_matcher *drop_matcher;
15946         struct mlx5_flow_cb_ctx ctx = {
15947                 .error = &error,
15948                 .data = &matcher,
15949         };
15950
15951         if (!priv->mtr_en || mtr_id_reg_c < 0) {
15952                 rte_errno = ENOTSUP;
15953                 return -1;
15954         }
15955         for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
15956                 if (!(domain_bitmap & (1 << domain)) ||
15957                         (mtrmng->def_rule[domain] && !fm->drop_cnt))
15958                         continue;
15959                 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15960                 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15961                 /* Create the drop table with METER DROP level. */
15962                 if (!mtrmng->drop_tbl[domain]) {
15963                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
15964                                         MLX5_FLOW_TABLE_LEVEL_METER,
15965                                         egress, transfer, false, NULL, 0,
15966                                         0, MLX5_MTR_TABLE_ID_DROP, &error);
15967                         if (!mtrmng->drop_tbl[domain]) {
15968                                 DRV_LOG(ERR, "Failed to create meter drop table.");
15969                                 goto policy_error;
15970                         }
15971                 }
15972                 /* Create default matcher in drop table. */
15973                 matcher.tbl = mtrmng->drop_tbl[domain],
15974                 tbl_data = container_of(mtrmng->drop_tbl[domain],
15975                                 struct mlx5_flow_tbl_data_entry, tbl);
15976                 if (!mtrmng->def_matcher[domain]) {
15977                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
15978                                        (enum modify_reg)mtr_id_reg_c,
15979                                        0, 0);
15980                         matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
15981                         matcher.crc = rte_raw_cksum
15982                                         ((const void *)matcher.mask.buf,
15983                                         matcher.mask.size);
15984                         entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
15985                         if (!entry) {
15986                                 DRV_LOG(ERR, "Failed to register meter "
15987                                 "drop default matcher.");
15988                                 goto policy_error;
15989                         }
15990                         mtrmng->def_matcher[domain] = container_of(entry,
15991                         struct mlx5_flow_dv_matcher, entry);
15992                 }
15993                 /* Create default rule in drop table. */
15994                 if (!mtrmng->def_rule[domain]) {
15995                         i = 0;
15996                         actions[i++] = priv->sh->dr_drop_action;
15997                         flow_dv_match_meta_reg(matcher_para.buf, value.buf,
15998                                 (enum modify_reg)mtr_id_reg_c, 0, 0);
15999                         ret = mlx5_flow_os_create_flow
16000                                 (mtrmng->def_matcher[domain]->matcher_object,
16001                                 (void *)&value, i, actions,
16002                                 &mtrmng->def_rule[domain]);
16003                         if (ret) {
16004                                 DRV_LOG(ERR, "Failed to create meter "
16005                                 "default drop rule for drop table.");
16006                                 goto policy_error;
16007                         }
16008                 }
16009                 if (!fm->drop_cnt)
16010                         continue;
16011                 MLX5_ASSERT(mtrmng->max_mtr_bits);
16012                 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
16013                         /* Create matchers for Drop. */
16014                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16015                                         (enum modify_reg)mtr_id_reg_c, 0,
16016                                         (mtr_id_mask << mtr_id_offset));
16017                         matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
16018                         matcher.crc = rte_raw_cksum
16019                                         ((const void *)matcher.mask.buf,
16020                                         matcher.mask.size);
16021                         entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
16022                         if (!entry) {
16023                                 DRV_LOG(ERR,
16024                                 "Failed to register meter drop matcher.");
16025                                 goto policy_error;
16026                         }
16027                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
16028                                 container_of(entry, struct mlx5_flow_dv_matcher,
16029                                              entry);
16030                 }
16031                 drop_matcher =
16032                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
16033                 /* Create drop rule, matching meter_id only. */
16034                 flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16035                                 (enum modify_reg)mtr_id_reg_c,
16036                                 (mtr_idx << mtr_id_offset), UINT32_MAX);
16037                 i = 0;
16038                 cnt = flow_dv_counter_get_by_idx(dev,
16039                                         fm->drop_cnt, NULL);
16040                 actions[i++] = cnt->action;
16041                 actions[i++] = priv->sh->dr_drop_action;
16042                 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
16043                                                (void *)&value, i, actions,
16044                                                &fm->drop_rule[domain]);
16045                 if (ret) {
16046                         DRV_LOG(ERR, "Failed to create meter "
16047                                 "drop rule for drop table.");
16048                                 goto policy_error;
16049                 }
16050         }
16051         return 0;
16052 policy_error:
16053         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16054                 if (fm->drop_rule[i]) {
16055                         claim_zero(mlx5_flow_os_destroy_flow
16056                                 (fm->drop_rule[i]));
16057                         fm->drop_rule[i] = NULL;
16058                 }
16059         }
16060         return -1;
16061 }
16062
16063 static struct mlx5_flow_meter_sub_policy *
16064 __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
16065                 struct mlx5_flow_meter_policy *mtr_policy,
16066                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS],
16067                 struct mlx5_flow_meter_sub_policy *next_sub_policy,
16068                 bool *is_reuse)
16069 {
16070         struct mlx5_priv *priv = dev->data->dev_private;
16071         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16072         uint32_t sub_policy_idx = 0;
16073         uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
16074         uint32_t i, j;
16075         struct mlx5_hrxq *hrxq;
16076         struct mlx5_flow_handle dh;
16077         struct mlx5_meter_policy_action_container *act_cnt;
16078         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16079         uint16_t sub_policy_num;
16080
16081         rte_spinlock_lock(&mtr_policy->sl);
16082         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16083                 if (!rss_desc[i])
16084                         continue;
16085                 hrxq_idx[i] = mlx5_hrxq_get(dev, rss_desc[i]);
16086                 if (!hrxq_idx[i]) {
16087                         rte_spinlock_unlock(&mtr_policy->sl);
16088                         return NULL;
16089                 }
16090         }
16091         sub_policy_num = (mtr_policy->sub_policy_num >>
16092                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16093                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16094         for (i = 0; i < sub_policy_num;
16095                 i++) {
16096                 for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) {
16097                         if (rss_desc[j] &&
16098                                 hrxq_idx[j] !=
16099                         mtr_policy->sub_policys[domain][i]->rix_hrxq[j])
16100                                 break;
16101                 }
16102                 if (j >= MLX5_MTR_RTE_COLORS) {
16103                         /*
16104                          * Found the sub policy table with
16105                          * the same queue per color
16106                          */
16107                         rte_spinlock_unlock(&mtr_policy->sl);
16108                         for (j = 0; j < MLX5_MTR_RTE_COLORS; j++)
16109                                 mlx5_hrxq_release(dev, hrxq_idx[j]);
16110                         *is_reuse = true;
16111                         return mtr_policy->sub_policys[domain][i];
16112                 }
16113         }
16114         /* Create sub policy. */
16115         if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
16116                 /* Reuse the first dummy sub_policy*/
16117                 sub_policy = mtr_policy->sub_policys[domain][0];
16118                 sub_policy_idx = sub_policy->idx;
16119         } else {
16120                 sub_policy = mlx5_ipool_zmalloc
16121                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16122                                 &sub_policy_idx);
16123                 if (!sub_policy ||
16124                         sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
16125                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
16126                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
16127                         goto rss_sub_policy_error;
16128                 }
16129                 sub_policy->idx = sub_policy_idx;
16130                 sub_policy->main_policy = mtr_policy;
16131         }
16132         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16133                 if (!rss_desc[i])
16134                         continue;
16135                 sub_policy->rix_hrxq[i] = hrxq_idx[i];
16136                 if (mtr_policy->is_hierarchy) {
16137                         act_cnt = &mtr_policy->act_cnt[i];
16138                         act_cnt->next_sub_policy = next_sub_policy;
16139                         mlx5_hrxq_release(dev, hrxq_idx[i]);
16140                 } else {
16141                         /*
16142                          * Overwrite the last action from
16143                          * RSS action to Queue action.
16144                          */
16145                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
16146                                 hrxq_idx[i]);
16147                         if (!hrxq) {
16148                                 DRV_LOG(ERR, "Failed to create policy hrxq");
16149                                 goto rss_sub_policy_error;
16150                         }
16151                         act_cnt = &mtr_policy->act_cnt[i];
16152                         if (act_cnt->rix_mark || act_cnt->modify_hdr) {
16153                                 memset(&dh, 0, sizeof(struct mlx5_flow_handle));
16154                                 if (act_cnt->rix_mark)
16155                                         dh.mark = 1;
16156                                 dh.fate_action = MLX5_FLOW_FATE_QUEUE;
16157                                 dh.rix_hrxq = hrxq_idx[i];
16158                                 flow_drv_rxq_flags_set(dev, &dh);
16159                         }
16160                 }
16161         }
16162         if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16163                 sub_policy, domain)) {
16164                 DRV_LOG(ERR, "Failed to create policy "
16165                         "rules per domain.");
16166                 goto rss_sub_policy_error;
16167         }
16168         if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16169                 i = (mtr_policy->sub_policy_num >>
16170                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16171                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16172                 mtr_policy->sub_policys[domain][i] = sub_policy;
16173                 i++;
16174                 if (i > MLX5_MTR_RSS_MAX_SUB_POLICY)
16175                         goto rss_sub_policy_error;
16176                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16177                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16178                 mtr_policy->sub_policy_num |=
16179                         (i & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16180                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16181         }
16182         rte_spinlock_unlock(&mtr_policy->sl);
16183         *is_reuse = false;
16184         return sub_policy;
16185 rss_sub_policy_error:
16186         if (sub_policy) {
16187                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16188                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16189                         i = (mtr_policy->sub_policy_num >>
16190                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16191                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16192                         mtr_policy->sub_policys[domain][i] = NULL;
16193                         mlx5_ipool_free
16194                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16195                                         sub_policy->idx);
16196                 }
16197         }
16198         rte_spinlock_unlock(&mtr_policy->sl);
16199         return NULL;
16200 }
16201
16202 /**
16203  * Find the policy table for prefix table with RSS.
16204  *
16205  * @param[in] dev
16206  *   Pointer to Ethernet device.
16207  * @param[in] mtr_policy
16208  *   Pointer to meter policy table.
16209  * @param[in] rss_desc
16210  *   Pointer to rss_desc
16211  * @return
16212  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
16213  */
16214 static struct mlx5_flow_meter_sub_policy *
16215 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
16216                 struct mlx5_flow_meter_policy *mtr_policy,
16217                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
16218 {
16219         struct mlx5_priv *priv = dev->data->dev_private;
16220         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16221         struct mlx5_flow_meter_info *next_fm;
16222         struct mlx5_flow_meter_policy *next_policy;
16223         struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL;
16224         struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM];
16225         struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM];
16226         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16227         bool reuse_sub_policy;
16228         uint32_t i = 0;
16229         uint32_t j = 0;
16230
16231         while (true) {
16232                 /* Iterate hierarchy to get all policies in this hierarchy. */
16233                 policies[i++] = mtr_policy;
16234                 if (!mtr_policy->is_hierarchy)
16235                         break;
16236                 if (i >= MLX5_MTR_CHAIN_MAX_NUM) {
16237                         DRV_LOG(ERR, "Exceed max meter number in hierarchy.");
16238                         return NULL;
16239                 }
16240                 next_fm = mlx5_flow_meter_find(priv,
16241                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16242                 if (!next_fm) {
16243                         DRV_LOG(ERR, "Failed to get next meter in hierarchy.");
16244                         return NULL;
16245                 }
16246                 next_policy =
16247                         mlx5_flow_meter_policy_find(dev, next_fm->policy_id,
16248                                                     NULL);
16249                 MLX5_ASSERT(next_policy);
16250                 mtr_policy = next_policy;
16251         }
16252         while (i) {
16253                 /**
16254                  * From last policy to the first one in hierarchy,
16255                  * create/get the sub policy for each of them.
16256                  */
16257                 sub_policy = __flow_dv_meter_get_rss_sub_policy(dev,
16258                                                         policies[--i],
16259                                                         rss_desc,
16260                                                         next_sub_policy,
16261                                                         &reuse_sub_policy);
16262                 if (!sub_policy) {
16263                         DRV_LOG(ERR, "Failed to get the sub policy.");
16264                         goto err_exit;
16265                 }
16266                 if (!reuse_sub_policy)
16267                         sub_policies[j++] = sub_policy;
16268                 next_sub_policy = sub_policy;
16269         }
16270         return sub_policy;
16271 err_exit:
16272         while (j) {
16273                 uint16_t sub_policy_num;
16274
16275                 sub_policy = sub_policies[--j];
16276                 mtr_policy = sub_policy->main_policy;
16277                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16278                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16279                         sub_policy_num = (mtr_policy->sub_policy_num >>
16280                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16281                                 MLX5_MTR_SUB_POLICY_NUM_MASK;
16282                         mtr_policy->sub_policys[domain][sub_policy_num - 1] =
16283                                                                         NULL;
16284                         sub_policy_num--;
16285                         mtr_policy->sub_policy_num &=
16286                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16287                                   (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
16288                         mtr_policy->sub_policy_num |=
16289                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16290                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
16291                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16292                                         sub_policy->idx);
16293                 }
16294         }
16295         return NULL;
16296 }
16297
16298 /**
16299  * Destroy the sub policy table with RX queue.
16300  *
16301  * @param[in] dev
16302  *   Pointer to Ethernet device.
16303  * @param[in] mtr_policy
16304  *   Pointer to meter policy table.
16305  */
16306 static void
16307 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
16308                 struct mlx5_flow_meter_policy *mtr_policy)
16309 {
16310         struct mlx5_priv *priv = dev->data->dev_private;
16311         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16312         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16313         uint32_t i, j;
16314         uint16_t sub_policy_num, new_policy_num;
16315
16316         rte_spinlock_lock(&mtr_policy->sl);
16317         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16318                 switch (mtr_policy->act_cnt[i].fate_action) {
16319                 case MLX5_FLOW_FATE_SHARED_RSS:
16320                         sub_policy_num = (mtr_policy->sub_policy_num >>
16321                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16322                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16323                         new_policy_num = sub_policy_num;
16324                         for (j = 0; j < sub_policy_num; j++) {
16325                                 sub_policy =
16326                                         mtr_policy->sub_policys[domain][j];
16327                                 if (sub_policy) {
16328                                         __flow_dv_destroy_sub_policy_rules(dev,
16329                                                 sub_policy);
16330                                 if (sub_policy !=
16331                                         mtr_policy->sub_policys[domain][0]) {
16332                                         mtr_policy->sub_policys[domain][j] =
16333                                                                 NULL;
16334                                         mlx5_ipool_free
16335                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16336                                                 sub_policy->idx);
16337                                                 new_policy_num--;
16338                                         }
16339                                 }
16340                         }
16341                         if (new_policy_num != sub_policy_num) {
16342                                 mtr_policy->sub_policy_num &=
16343                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16344                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16345                                 mtr_policy->sub_policy_num |=
16346                                 (new_policy_num &
16347                                         MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16348                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16349                         }
16350                         break;
16351                 case MLX5_FLOW_FATE_QUEUE:
16352                         sub_policy = mtr_policy->sub_policys[domain][0];
16353                         __flow_dv_destroy_sub_policy_rules(dev,
16354                                                 sub_policy);
16355                         break;
16356                 default:
16357                         /*Other actions without queue and do nothing*/
16358                         break;
16359                 }
16360         }
16361         rte_spinlock_unlock(&mtr_policy->sl);
16362 }
16363
16364 /**
16365  * Validate the batch counter support in root table.
16366  *
16367  * Create a simple flow with invalid counter and drop action on root table to
16368  * validate if batch counter with offset on root table is supported or not.
16369  *
16370  * @param[in] dev
16371  *   Pointer to rte_eth_dev structure.
16372  *
16373  * @return
16374  *   0 on success, a negative errno value otherwise and rte_errno is set.
16375  */
16376 int
16377 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
16378 {
16379         struct mlx5_priv *priv = dev->data->dev_private;
16380         struct mlx5_dev_ctx_shared *sh = priv->sh;
16381         struct mlx5_flow_dv_match_params mask = {
16382                 .size = sizeof(mask.buf),
16383         };
16384         struct mlx5_flow_dv_match_params value = {
16385                 .size = sizeof(value.buf),
16386         };
16387         struct mlx5dv_flow_matcher_attr dv_attr = {
16388                 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS,
16389                 .priority = 0,
16390                 .match_criteria_enable = 0,
16391                 .match_mask = (void *)&mask,
16392         };
16393         void *actions[2] = { 0 };
16394         struct mlx5_flow_tbl_resource *tbl = NULL;
16395         struct mlx5_devx_obj *dcs = NULL;
16396         void *matcher = NULL;
16397         void *flow = NULL;
16398         int ret = -1;
16399
16400         tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL,
16401                                         0, 0, 0, NULL);
16402         if (!tbl)
16403                 goto err;
16404         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
16405         if (!dcs)
16406                 goto err;
16407         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
16408                                                     &actions[0]);
16409         if (ret)
16410                 goto err;
16411         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
16412         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
16413                                                &matcher);
16414         if (ret)
16415                 goto err;
16416         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
16417                                        actions, &flow);
16418 err:
16419         /*
16420          * If batch counter with offset is not supported, the driver will not
16421          * validate the invalid offset value, flow create should success.
16422          * In this case, it means batch counter is not supported in root table.
16423          *
16424          * Otherwise, if flow create is failed, counter offset is supported.
16425          */
16426         if (flow) {
16427                 DRV_LOG(INFO, "Batch counter is not supported in root "
16428                               "table. Switch to fallback mode.");
16429                 rte_errno = ENOTSUP;
16430                 ret = -rte_errno;
16431                 claim_zero(mlx5_flow_os_destroy_flow(flow));
16432         } else {
16433                 /* Check matcher to make sure validate fail at flow create. */
16434                 if (!matcher || (matcher && errno != EINVAL))
16435                         DRV_LOG(ERR, "Unexpected error in counter offset "
16436                                      "support detection");
16437                 ret = 0;
16438         }
16439         if (actions[0])
16440                 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0]));
16441         if (matcher)
16442                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
16443         if (tbl)
16444                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
16445         if (dcs)
16446                 claim_zero(mlx5_devx_cmd_destroy(dcs));
16447         return ret;
16448 }
16449
16450 /**
16451  * Query a devx counter.
16452  *
16453  * @param[in] dev
16454  *   Pointer to the Ethernet device structure.
16455  * @param[in] cnt
16456  *   Index to the flow counter.
16457  * @param[in] clear
16458  *   Set to clear the counter statistics.
16459  * @param[out] pkts
16460  *   The statistics value of packets.
16461  * @param[out] bytes
16462  *   The statistics value of bytes.
16463  *
16464  * @return
16465  *   0 on success, otherwise return -1.
16466  */
16467 static int
16468 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
16469                       uint64_t *pkts, uint64_t *bytes)
16470 {
16471         struct mlx5_priv *priv = dev->data->dev_private;
16472         struct mlx5_flow_counter *cnt;
16473         uint64_t inn_pkts, inn_bytes;
16474         int ret;
16475
16476         if (!priv->config.devx)
16477                 return -1;
16478
16479         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
16480         if (ret)
16481                 return -1;
16482         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
16483         *pkts = inn_pkts - cnt->hits;
16484         *bytes = inn_bytes - cnt->bytes;
16485         if (clear) {
16486                 cnt->hits = inn_pkts;
16487                 cnt->bytes = inn_bytes;
16488         }
16489         return 0;
16490 }
16491
16492 /**
16493  * Get aged-out flows.
16494  *
16495  * @param[in] dev
16496  *   Pointer to the Ethernet device structure.
16497  * @param[in] context
16498  *   The address of an array of pointers to the aged-out flows contexts.
16499  * @param[in] nb_contexts
16500  *   The length of context array pointers.
16501  * @param[out] error
16502  *   Perform verbose error reporting if not NULL. Initialized in case of
16503  *   error only.
16504  *
16505  * @return
16506  *   how many contexts get in success, otherwise negative errno value.
16507  *   if nb_contexts is 0, return the amount of all aged contexts.
16508  *   if nb_contexts is not 0 , return the amount of aged flows reported
16509  *   in the context array.
16510  * @note: only stub for now
16511  */
16512 static int
16513 flow_get_aged_flows(struct rte_eth_dev *dev,
16514                     void **context,
16515                     uint32_t nb_contexts,
16516                     struct rte_flow_error *error)
16517 {
16518         struct mlx5_priv *priv = dev->data->dev_private;
16519         struct mlx5_age_info *age_info;
16520         struct mlx5_age_param *age_param;
16521         struct mlx5_flow_counter *counter;
16522         struct mlx5_aso_age_action *act;
16523         int nb_flows = 0;
16524
16525         if (nb_contexts && !context)
16526                 return rte_flow_error_set(error, EINVAL,
16527                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16528                                           NULL, "empty context");
16529         age_info = GET_PORT_AGE_INFO(priv);
16530         rte_spinlock_lock(&age_info->aged_sl);
16531         LIST_FOREACH(act, &age_info->aged_aso, next) {
16532                 nb_flows++;
16533                 if (nb_contexts) {
16534                         context[nb_flows - 1] =
16535                                                 act->age_params.context;
16536                         if (!(--nb_contexts))
16537                                 break;
16538                 }
16539         }
16540         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
16541                 nb_flows++;
16542                 if (nb_contexts) {
16543                         age_param = MLX5_CNT_TO_AGE(counter);
16544                         context[nb_flows - 1] = age_param->context;
16545                         if (!(--nb_contexts))
16546                                 break;
16547                 }
16548         }
16549         rte_spinlock_unlock(&age_info->aged_sl);
16550         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
16551         return nb_flows;
16552 }
16553
16554 /*
16555  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
16556  */
16557 static uint32_t
16558 flow_dv_counter_allocate(struct rte_eth_dev *dev)
16559 {
16560         return flow_dv_counter_alloc(dev, 0);
16561 }
16562
16563 /**
16564  * Validate indirect action.
16565  * Dispatcher for action type specific validation.
16566  *
16567  * @param[in] dev
16568  *   Pointer to the Ethernet device structure.
16569  * @param[in] conf
16570  *   Indirect action configuration.
16571  * @param[in] action
16572  *   The indirect action object to validate.
16573  * @param[out] error
16574  *   Perform verbose error reporting if not NULL. Initialized in case of
16575  *   error only.
16576  *
16577  * @return
16578  *   0 on success, otherwise negative errno value.
16579  */
16580 static int
16581 flow_dv_action_validate(struct rte_eth_dev *dev,
16582                         const struct rte_flow_indir_action_conf *conf,
16583                         const struct rte_flow_action *action,
16584                         struct rte_flow_error *err)
16585 {
16586         struct mlx5_priv *priv = dev->data->dev_private;
16587
16588         RTE_SET_USED(conf);
16589         switch (action->type) {
16590         case RTE_FLOW_ACTION_TYPE_RSS:
16591                 /*
16592                  * priv->obj_ops is set according to driver capabilities.
16593                  * When DevX capabilities are
16594                  * sufficient, it is set to devx_obj_ops.
16595                  * Otherwise, it is set to ibv_obj_ops.
16596                  * ibv_obj_ops doesn't support ind_table_modify operation.
16597                  * In this case the indirect RSS action can't be used.
16598                  */
16599                 if (priv->obj_ops.ind_table_modify == NULL)
16600                         return rte_flow_error_set
16601                                         (err, ENOTSUP,
16602                                          RTE_FLOW_ERROR_TYPE_ACTION,
16603                                          NULL,
16604                                          "Indirect RSS action not supported");
16605                 return mlx5_validate_action_rss(dev, action, err);
16606         case RTE_FLOW_ACTION_TYPE_AGE:
16607                 if (!priv->sh->aso_age_mng)
16608                         return rte_flow_error_set(err, ENOTSUP,
16609                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16610                                                 NULL,
16611                                                 "Indirect age action not supported");
16612                 return flow_dv_validate_action_age(0, action, dev, err);
16613         case RTE_FLOW_ACTION_TYPE_COUNT:
16614                 /*
16615                  * There are two mechanisms to share the action count.
16616                  * The old mechanism uses the shared field to share, while the
16617                  * new mechanism uses the indirect action API.
16618                  * This validation comes to make sure that the two mechanisms
16619                  * are not combined.
16620                  */
16621                 if (is_shared_action_count(action))
16622                         return rte_flow_error_set(err, ENOTSUP,
16623                                                   RTE_FLOW_ERROR_TYPE_ACTION,
16624                                                   NULL,
16625                                                   "Mix shared and indirect counter is not supported");
16626                 return flow_dv_validate_action_count(dev, true, 0, err);
16627         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
16628                 if (!priv->sh->ct_aso_en)
16629                         return rte_flow_error_set(err, ENOTSUP,
16630                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
16631                                         "ASO CT is not supported");
16632                 return mlx5_validate_action_ct(dev, action->conf, err);
16633         default:
16634                 return rte_flow_error_set(err, ENOTSUP,
16635                                           RTE_FLOW_ERROR_TYPE_ACTION,
16636                                           NULL,
16637                                           "action type not supported");
16638         }
16639 }
16640
16641 /**
16642  * Validate meter policy actions.
16643  * Dispatcher for action type specific validation.
16644  *
16645  * @param[in] dev
16646  *   Pointer to the Ethernet device structure.
16647  * @param[in] action
16648  *   The meter policy action object to validate.
16649  * @param[in] attr
16650  *   Attributes of flow to determine steering domain.
16651  * @param[out] error
16652  *   Perform verbose error reporting if not NULL. Initialized in case of
16653  *   error only.
16654  *
16655  * @return
16656  *   0 on success, otherwise negative errno value.
16657  */
16658 static int
16659 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
16660                         const struct rte_flow_action *actions[RTE_COLORS],
16661                         struct rte_flow_attr *attr,
16662                         bool *is_rss,
16663                         uint8_t *domain_bitmap,
16664                         bool *is_def_policy,
16665                         struct rte_mtr_error *error)
16666 {
16667         struct mlx5_priv *priv = dev->data->dev_private;
16668         struct mlx5_dev_config *dev_conf = &priv->config;
16669         const struct rte_flow_action *act;
16670         uint64_t action_flags = 0;
16671         int actions_n;
16672         int i, ret;
16673         struct rte_flow_error flow_err;
16674         uint8_t domain_color[RTE_COLORS] = {0};
16675         uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
16676
16677         if (!priv->config.dv_esw_en)
16678                 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
16679         *domain_bitmap = def_domain;
16680         if (actions[RTE_COLOR_YELLOW] &&
16681                 actions[RTE_COLOR_YELLOW]->type != RTE_FLOW_ACTION_TYPE_END)
16682                 return -rte_mtr_error_set(error, ENOTSUP,
16683                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
16684                                 NULL,
16685                                 "Yellow color does not support any action.");
16686         if (actions[RTE_COLOR_YELLOW] &&
16687                 actions[RTE_COLOR_YELLOW]->type != RTE_FLOW_ACTION_TYPE_DROP)
16688                 return -rte_mtr_error_set(error, ENOTSUP,
16689                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
16690                                 NULL, "Red color only supports drop action.");
16691         /*
16692          * Check default policy actions:
16693          * Green/Yellow: no action, Red: drop action
16694          */
16695         if ((!actions[RTE_COLOR_GREEN] ||
16696                 actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END)) {
16697                 *is_def_policy = true;
16698                 return 0;
16699         }
16700         flow_err.message = NULL;
16701         for (i = 0; i < RTE_COLORS; i++) {
16702                 act = actions[i];
16703                 for (action_flags = 0, actions_n = 0;
16704                         act && act->type != RTE_FLOW_ACTION_TYPE_END;
16705                         act++) {
16706                         if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
16707                                 return -rte_mtr_error_set(error, ENOTSUP,
16708                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16709                                           NULL, "too many actions");
16710                         switch (act->type) {
16711                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
16712                                 if (!priv->config.dv_esw_en)
16713                                         return -rte_mtr_error_set(error,
16714                                         ENOTSUP,
16715                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16716                                         NULL, "PORT action validate check"
16717                                         " fail for ESW disable");
16718                                 ret = flow_dv_validate_action_port_id(dev,
16719                                                 action_flags,
16720                                                 act, attr, &flow_err);
16721                                 if (ret)
16722                                         return -rte_mtr_error_set(error,
16723                                         ENOTSUP,
16724                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16725                                         NULL, flow_err.message ?
16726                                         flow_err.message :
16727                                         "PORT action validate check fail");
16728                                 ++actions_n;
16729                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
16730                                 break;
16731                         case RTE_FLOW_ACTION_TYPE_MARK:
16732                                 ret = flow_dv_validate_action_mark(dev, act,
16733                                                            action_flags,
16734                                                            attr, &flow_err);
16735                                 if (ret < 0)
16736                                         return -rte_mtr_error_set(error,
16737                                         ENOTSUP,
16738                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16739                                         NULL, flow_err.message ?
16740                                         flow_err.message :
16741                                         "Mark action validate check fail");
16742                                 if (dev_conf->dv_xmeta_en !=
16743                                         MLX5_XMETA_MODE_LEGACY)
16744                                         return -rte_mtr_error_set(error,
16745                                         ENOTSUP,
16746                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16747                                         NULL, "Extend MARK action is "
16748                                         "not supported. Please try use "
16749                                         "default policy for meter.");
16750                                 action_flags |= MLX5_FLOW_ACTION_MARK;
16751                                 ++actions_n;
16752                                 break;
16753                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
16754                                 ret = flow_dv_validate_action_set_tag(dev,
16755                                                         act, action_flags,
16756                                                         attr, &flow_err);
16757                                 if (ret)
16758                                         return -rte_mtr_error_set(error,
16759                                         ENOTSUP,
16760                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16761                                         NULL, flow_err.message ?
16762                                         flow_err.message :
16763                                         "Set tag action validate check fail");
16764                                 /*
16765                                  * Count all modify-header actions
16766                                  * as one action.
16767                                  */
16768                                 if (!(action_flags &
16769                                         MLX5_FLOW_MODIFY_HDR_ACTIONS))
16770                                         ++actions_n;
16771                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
16772                                 break;
16773                         case RTE_FLOW_ACTION_TYPE_DROP:
16774                                 ret = mlx5_flow_validate_action_drop
16775                                         (action_flags,
16776                                         attr, &flow_err);
16777                                 if (ret < 0)
16778                                         return -rte_mtr_error_set(error,
16779                                         ENOTSUP,
16780                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16781                                         NULL, flow_err.message ?
16782                                         flow_err.message :
16783                                         "Drop action validate check fail");
16784                                 action_flags |= MLX5_FLOW_ACTION_DROP;
16785                                 ++actions_n;
16786                                 break;
16787                         case RTE_FLOW_ACTION_TYPE_QUEUE:
16788                                 /*
16789                                  * Check whether extensive
16790                                  * metadata feature is engaged.
16791                                  */
16792                                 if (dev_conf->dv_flow_en &&
16793                                         (dev_conf->dv_xmeta_en !=
16794                                         MLX5_XMETA_MODE_LEGACY) &&
16795                                         mlx5_flow_ext_mreg_supported(dev))
16796                                         return -rte_mtr_error_set(error,
16797                                           ENOTSUP,
16798                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16799                                           NULL, "Queue action with meta "
16800                                           "is not supported. Please try use "
16801                                           "default policy for meter.");
16802                                 ret = mlx5_flow_validate_action_queue(act,
16803                                                         action_flags, dev,
16804                                                         attr, &flow_err);
16805                                 if (ret < 0)
16806                                         return -rte_mtr_error_set(error,
16807                                           ENOTSUP,
16808                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16809                                           NULL, flow_err.message ?
16810                                           flow_err.message :
16811                                           "Queue action validate check fail");
16812                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
16813                                 ++actions_n;
16814                                 break;
16815                         case RTE_FLOW_ACTION_TYPE_RSS:
16816                                 if (dev_conf->dv_flow_en &&
16817                                         (dev_conf->dv_xmeta_en !=
16818                                         MLX5_XMETA_MODE_LEGACY) &&
16819                                         mlx5_flow_ext_mreg_supported(dev))
16820                                         return -rte_mtr_error_set(error,
16821                                           ENOTSUP,
16822                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16823                                           NULL, "RSS action with meta "
16824                                           "is not supported. Please try use "
16825                                           "default policy for meter.");
16826                                 ret = mlx5_validate_action_rss(dev, act,
16827                                                 &flow_err);
16828                                 if (ret < 0)
16829                                         return -rte_mtr_error_set(error,
16830                                           ENOTSUP,
16831                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16832                                           NULL, flow_err.message ?
16833                                           flow_err.message :
16834                                           "RSS action validate check fail");
16835                                 action_flags |= MLX5_FLOW_ACTION_RSS;
16836                                 ++actions_n;
16837                                 *is_rss = true;
16838                                 break;
16839                         case RTE_FLOW_ACTION_TYPE_JUMP:
16840                                 ret = flow_dv_validate_action_jump(dev,
16841                                         NULL, act, action_flags,
16842                                         attr, true, &flow_err);
16843                                 if (ret)
16844                                         return -rte_mtr_error_set(error,
16845                                           ENOTSUP,
16846                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16847                                           NULL, flow_err.message ?
16848                                           flow_err.message :
16849                                           "Jump action validate check fail");
16850                                 ++actions_n;
16851                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
16852                                 break;
16853                         default:
16854                                 return -rte_mtr_error_set(error, ENOTSUP,
16855                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16856                                         NULL,
16857                                         "Doesn't support optional action");
16858                         }
16859                 }
16860                 /* Yellow is not supported, just skip. */
16861                 if (i == RTE_COLOR_YELLOW)
16862                         continue;
16863                 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
16864                         domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT;
16865                 else if ((action_flags &
16866                         (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) ||
16867                         (action_flags & MLX5_FLOW_ACTION_MARK))
16868                         /*
16869                          * Only support MLX5_XMETA_MODE_LEGACY
16870                          * so MARK action only in ingress domain.
16871                          */
16872                         domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT;
16873                 else
16874                         domain_color[i] = def_domain;
16875                 /*
16876                  * Validate the drop action mutual exclusion
16877                  * with other actions. Drop action is mutually-exclusive
16878                  * with any other action, except for Count action.
16879                  */
16880                 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
16881                         (action_flags & ~MLX5_FLOW_ACTION_DROP)) {
16882                         return -rte_mtr_error_set(error, ENOTSUP,
16883                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
16884                                 NULL, "Drop action is mutually-exclusive "
16885                                 "with any other action");
16886                 }
16887                 /* Eswitch has few restrictions on using items and actions */
16888                 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) {
16889                         if (!mlx5_flow_ext_mreg_supported(dev) &&
16890                                 action_flags & MLX5_FLOW_ACTION_MARK)
16891                                 return -rte_mtr_error_set(error, ENOTSUP,
16892                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16893                                         NULL, "unsupported action MARK");
16894                         if (action_flags & MLX5_FLOW_ACTION_QUEUE)
16895                                 return -rte_mtr_error_set(error, ENOTSUP,
16896                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16897                                         NULL, "unsupported action QUEUE");
16898                         if (action_flags & MLX5_FLOW_ACTION_RSS)
16899                                 return -rte_mtr_error_set(error, ENOTSUP,
16900                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16901                                         NULL, "unsupported action RSS");
16902                         if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
16903                                 return -rte_mtr_error_set(error, ENOTSUP,
16904                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16905                                         NULL, "no fate action is found");
16906                 } else {
16907                         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) &&
16908                                 (domain_color[i] &
16909                                 MLX5_MTR_DOMAIN_INGRESS_BIT)) {
16910                                 if ((domain_color[i] &
16911                                         MLX5_MTR_DOMAIN_EGRESS_BIT))
16912                                         domain_color[i] =
16913                                         MLX5_MTR_DOMAIN_EGRESS_BIT;
16914                                 else
16915                                         return -rte_mtr_error_set(error,
16916                                         ENOTSUP,
16917                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
16918                                         NULL, "no fate action is found");
16919                         }
16920                 }
16921                 if (domain_color[i] != def_domain)
16922                         *domain_bitmap = domain_color[i];
16923         }
16924         return 0;
16925 }
16926
16927 static int
16928 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
16929 {
16930         struct mlx5_priv *priv = dev->data->dev_private;
16931         int ret = 0;
16932
16933         if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
16934                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain,
16935                                                 flags);
16936                 if (ret != 0)
16937                         return ret;
16938         }
16939         if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
16940                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags);
16941                 if (ret != 0)
16942                         return ret;
16943         }
16944         if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
16945                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags);
16946                 if (ret != 0)
16947                         return ret;
16948         }
16949         return 0;
16950 }
16951
16952 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
16953         .validate = flow_dv_validate,
16954         .prepare = flow_dv_prepare,
16955         .translate = flow_dv_translate,
16956         .apply = flow_dv_apply,
16957         .remove = flow_dv_remove,
16958         .destroy = flow_dv_destroy,
16959         .query = flow_dv_query,
16960         .create_mtr_tbls = flow_dv_create_mtr_tbls,
16961         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls,
16962         .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls,
16963         .create_meter = flow_dv_mtr_alloc,
16964         .free_meter = flow_dv_aso_mtr_release_to_pool,
16965         .validate_mtr_acts = flow_dv_validate_mtr_policy_acts,
16966         .create_mtr_acts = flow_dv_create_mtr_policy_acts,
16967         .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts,
16968         .create_policy_rules = flow_dv_create_policy_rules,
16969         .destroy_policy_rules = flow_dv_destroy_policy_rules,
16970         .create_def_policy = flow_dv_create_def_policy,
16971         .destroy_def_policy = flow_dv_destroy_def_policy,
16972         .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare,
16973         .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq,
16974         .counter_alloc = flow_dv_counter_allocate,
16975         .counter_free = flow_dv_counter_free,
16976         .counter_query = flow_dv_counter_query,
16977         .get_aged_flows = flow_get_aged_flows,
16978         .action_validate = flow_dv_action_validate,
16979         .action_create = flow_dv_action_create,
16980         .action_destroy = flow_dv_action_destroy,
16981         .action_update = flow_dv_action_update,
16982         .action_query = flow_dv_action_query,
16983         .sync_domain = flow_dv_sync_domain,
16984 };
16985
16986 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
16987