drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24 #include <rte_mtr.h>
  25 #include <rte_mtr_driver.h>
  26 #include <rte_tailq.h>
  27
  28 #include <mlx5_glue.h>
  29 #include <mlx5_devx_cmds.h>
  30 #include <mlx5_prm.h>
  31 #include <mlx5_malloc.h>
  32
  33 #include "mlx5_defs.h"
  34 #include "mlx5.h"
  35 #include "mlx5_common_os.h"
  36 #include "mlx5_flow.h"
  37 #include "mlx5_flow_os.h"
  38 #include "mlx5_rx.h"
  39 #include "mlx5_tx.h"
  40 #include "rte_pmd_mlx5.h"
  41
  42 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  43
  44 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  45 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  46 #endif
  47
  48 #ifndef HAVE_MLX5DV_DR_ESWITCH
  49 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  50 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  51 #endif
  52 #endif
  53
  54 #ifndef HAVE_MLX5DV_DR
  55 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  56 #endif
  57
  58 /* VLAN header definitions */
  59 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  60 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  61 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  62 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  63 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  64
  65 union flow_dv_attr {
  66         struct {
  67                 uint32_t valid:1;
  68                 uint32_t ipv4:1;
  69                 uint32_t ipv6:1;
  70                 uint32_t tcp:1;
  71                 uint32_t udp:1;
  72                 uint32_t reserved:27;
  73         };
  74         uint32_t attr;
  75 };
  76
  77 static int
  78 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
  79                              struct mlx5_flow_tbl_resource *tbl);
  80
  81 static int
  82 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  83                                      uint32_t encap_decap_idx);
  84
  85 static int
  86 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  87                                         uint32_t port_id);
  88 static void
  89 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);
  90
  91 static int
  92 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
  93                                   uint32_t rix_jump);
  94
  95 /**
  96  * Initialize flow attributes structure according to flow items' types.
  97  *
  98  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  99  * mode. For tunnel mode, the items to be modified are the outermost ones.
 100  *
 101  * @param[in] item
 102  *   Pointer to item specification.
 103  * @param[out] attr
 104  *   Pointer to flow attributes structure.
 105  * @param[in] dev_flow
 106  *   Pointer to the sub flow.
 107  * @param[in] tunnel_decap
 108  *   Whether action is after tunnel decapsulation.
 109  */
 110 static void
 111 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 112                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 113 {
 114         uint64_t layers = dev_flow->handle->layers;
 115
 116         /*
 117          * If layers is already initialized, it means this dev_flow is the
 118          * suffix flow, the layers flags is set by the prefix flow. Need to
 119          * use the layer flags from prefix flow as the suffix flow may not
 120          * have the user defined items as the flow is split.
 121          */
 122         if (layers) {
 123                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 124                         attr->ipv4 = 1;
 125                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 126                         attr->ipv6 = 1;
 127                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 128                         attr->tcp = 1;
 129                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 130                         attr->udp = 1;
 131                 attr->valid = 1;
 132                 return;
 133         }
 134         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 135                 uint8_t next_protocol = 0xff;
 136                 switch (item->type) {
 137                 case RTE_FLOW_ITEM_TYPE_GRE:
 138                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 139                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 140                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 141                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 142                 case RTE_FLOW_ITEM_TYPE_MPLS:
 143                         if (tunnel_decap)
 144                                 attr->attr = 0;
 145                         break;
 146                 case RTE_FLOW_ITEM_TYPE_IPV4:
 147                         if (!attr->ipv6)
 148                                 attr->ipv4 = 1;
 149                         if (item->mask != NULL &&
 150                             ((const struct rte_flow_item_ipv4 *)
 151                             item->mask)->hdr.next_proto_id)
 152                                 next_protocol =
 153                                     ((const struct rte_flow_item_ipv4 *)
 154                                       (item->spec))->hdr.next_proto_id &
 155                                     ((const struct rte_flow_item_ipv4 *)
 156                                       (item->mask))->hdr.next_proto_id;
 157                         if ((next_protocol == IPPROTO_IPIP ||
 158                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 159                                 attr->attr = 0;
 160                         break;
 161                 case RTE_FLOW_ITEM_TYPE_IPV6:
 162                         if (!attr->ipv4)
 163                                 attr->ipv6 = 1;
 164                         if (item->mask != NULL &&
 165                             ((const struct rte_flow_item_ipv6 *)
 166                             item->mask)->hdr.proto)
 167                                 next_protocol =
 168                                     ((const struct rte_flow_item_ipv6 *)
 169                                       (item->spec))->hdr.proto &
 170                                     ((const struct rte_flow_item_ipv6 *)
 171                                       (item->mask))->hdr.proto;
 172                         if ((next_protocol == IPPROTO_IPIP ||
 173                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 174                                 attr->attr = 0;
 175                         break;
 176                 case RTE_FLOW_ITEM_TYPE_UDP:
 177                         if (!attr->tcp)
 178                                 attr->udp = 1;
 179                         break;
 180                 case RTE_FLOW_ITEM_TYPE_TCP:
 181                         if (!attr->udp)
 182                                 attr->tcp = 1;
 183                         break;
 184                 default:
 185                         break;
 186                 }
 187         }
 188         attr->valid = 1;
 189 }
 190
 191 /**
 192  * Convert rte_mtr_color to mlx5 color.
 193  *
 194  * @param[in] rcol
 195  *   rte_mtr_color.
 196  *
 197  * @return
 198  *   mlx5 color.
 199  */
 200 static int
 201 rte_col_2_mlx5_col(enum rte_color rcol)
 202 {
 203         switch (rcol) {
 204         case RTE_COLOR_GREEN:
 205                 return MLX5_FLOW_COLOR_GREEN;
 206         case RTE_COLOR_YELLOW:
 207                 return MLX5_FLOW_COLOR_YELLOW;
 208         case RTE_COLOR_RED:
 209                 return MLX5_FLOW_COLOR_RED;
 210         default:
 211                 break;
 212         }
 213         return MLX5_FLOW_COLOR_UNDEFINED;
 214 }
 215
 216 struct field_modify_info {
 217         uint32_t size; /* Size of field in protocol header, in bytes. */
 218         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 219         enum mlx5_modification_field id;
 220 };
 221
 222 struct field_modify_info modify_eth[] = {
 223         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 224         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 225         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 226         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 227         {0, 0, 0},
 228 };
 229
 230 struct field_modify_info modify_vlan_out_first_vid[] = {
 231         /* Size in bits !!! */
 232         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv4[] = {
 237         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 239         {4, 12, MLX5_MODI_OUT_SIPV4},
 240         {4, 16, MLX5_MODI_OUT_DIPV4},
 241         {0, 0, 0},
 242 };
 243
 244 struct field_modify_info modify_ipv6[] = {
 245         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 246         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 247         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 248         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 249         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 250         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 251         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 252         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 253         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 254         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 255         {0, 0, 0},
 256 };
 257
 258 struct field_modify_info modify_udp[] = {
 259         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 260         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 261         {0, 0, 0},
 262 };
 263
 264 struct field_modify_info modify_tcp[] = {
 265         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 266         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 267         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 268         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 269         {0, 0, 0},
 270 };
 271
 272 static const struct rte_flow_item *
 273 mlx5_flow_find_tunnel_item(const struct rte_flow_item *item)
 274 {
 275         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 276                 switch (item->type) {
 277                 default:
 278                         break;
 279                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 280                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 281                 case RTE_FLOW_ITEM_TYPE_GRE:
 282                 case RTE_FLOW_ITEM_TYPE_MPLS:
 283                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 284                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 285                         return item;
 286                 case RTE_FLOW_ITEM_TYPE_IPV4:
 287                 case RTE_FLOW_ITEM_TYPE_IPV6:
 288                         if (item[1].type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 289                             item[1].type == RTE_FLOW_ITEM_TYPE_IPV6)
 290                                 return item;
 291                         break;
 292                 }
 293         }
 294         return NULL;
 295 }
 296
 297 static void
 298 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 299                           uint8_t next_protocol, uint64_t *item_flags,
 300                           int *tunnel)
 301 {
 302         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 303                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 304         if (next_protocol == IPPROTO_IPIP) {
 305                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 306                 *tunnel = 1;
 307         }
 308         if (next_protocol == IPPROTO_IPV6) {
 309                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 310                 *tunnel = 1;
 311         }
 312 }
 313
 314 /* Update VLAN's VID/PCP based on input rte_flow_action.
 315  *
 316  * @param[in] action
 317  *   Pointer to struct rte_flow_action.
 318  * @param[out] vlan
 319  *   Pointer to struct rte_vlan_hdr.
 320  */
 321 static void
 322 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 323                          struct rte_vlan_hdr *vlan)
 324 {
 325         uint16_t vlan_tci;
 326         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 327                 vlan_tci =
 328                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 329                                                action->conf)->vlan_pcp;
 330                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 331                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 332                 vlan->vlan_tci |= vlan_tci;
 333         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 334                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 335                 vlan->vlan_tci |= rte_be_to_cpu_16
 336                     (((const struct rte_flow_action_of_set_vlan_vid *)
 337                                              action->conf)->vlan_vid);
 338         }
 339 }
 340
 341 /**
 342  * Fetch 1, 2, 3 or 4 byte field from the byte array
 343  * and return as unsigned integer in host-endian format.
 344  *
 345  * @param[in] data
 346  *   Pointer to data array.
 347  * @param[in] size
 348  *   Size of field to extract.
 349  *
 350  * @return
 351  *   converted field in host endian format.
 352  */
 353 static inline uint32_t
 354 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 355 {
 356         uint32_t ret;
 357
 358         switch (size) {
 359         case 1:
 360                 ret = *data;
 361                 break;
 362         case 2:
 363                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 364                 break;
 365         case 3:
 366                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 367                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 368                 break;
 369         case 4:
 370                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 371                 break;
 372         default:
 373                 MLX5_ASSERT(false);
 374                 ret = 0;
 375                 break;
 376         }
 377         return ret;
 378 }
 379
 380 /**
 381  * Convert modify-header action to DV specification.
 382  *
 383  * Data length of each action is determined by provided field description
 384  * and the item mask. Data bit offset and width of each action is determined
 385  * by provided item mask.
 386  *
 387  * @param[in] item
 388  *   Pointer to item specification.
 389  * @param[in] field
 390  *   Pointer to field modification information.
 391  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 392  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 393  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 394  * @param[in] dcopy
 395  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 396  *   Negative offset value sets the same offset as source offset.
 397  *   size field is ignored, value is taken from source field.
 398  * @param[in,out] resource
 399  *   Pointer to the modify-header resource.
 400  * @param[in] type
 401  *   Type of modification.
 402  * @param[out] error
 403  *   Pointer to the error structure.
 404  *
 405  * @return
 406  *   0 on success, a negative errno value otherwise and rte_errno is set.
 407  */
 408 static int
 409 flow_dv_convert_modify_action(struct rte_flow_item *item,
 410                               struct field_modify_info *field,
 411                               struct field_modify_info *dcopy,
 412                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 413                               uint32_t type, struct rte_flow_error *error)
 414 {
 415         uint32_t i = resource->actions_num;
 416         struct mlx5_modification_cmd *actions = resource->actions;
 417         uint32_t carry_b = 0;
 418
 419         /*
 420          * The item and mask are provided in big-endian format.
 421          * The fields should be presented as in big-endian format either.
 422          * Mask must be always present, it defines the actual field width.
 423          */
 424         MLX5_ASSERT(item->mask);
 425         MLX5_ASSERT(field->size);
 426         do {
 427                 uint32_t size_b;
 428                 uint32_t off_b;
 429                 uint32_t mask;
 430                 uint32_t data;
 431                 bool next_field = true;
 432                 bool next_dcopy = true;
 433
 434                 if (i >= MLX5_MAX_MODIFY_NUM)
 435                         return rte_flow_error_set(error, EINVAL,
 436                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 437                                  "too many items to modify");
 438                 /* Fetch variable byte size mask from the array. */
 439                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 440                                            field->offset, field->size);
 441                 if (!mask) {
 442                         ++field;
 443                         continue;
 444                 }
 445                 /* Deduce actual data width in bits from mask value. */
 446                 off_b = rte_bsf32(mask) + carry_b;
 447                 size_b = sizeof(uint32_t) * CHAR_BIT -
 448                          off_b - __builtin_clz(mask);
 449                 MLX5_ASSERT(size_b);
 450                 actions[i] = (struct mlx5_modification_cmd) {
 451                         .action_type = type,
 452                         .field = field->id,
 453                         .offset = off_b,
 454                         .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ?
 455                                 0 : size_b,
 456                 };
 457                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 458                         MLX5_ASSERT(dcopy);
 459                         actions[i].dst_field = dcopy->id;
 460                         actions[i].dst_offset =
 461                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 462                         /* Convert entire record to big-endian format. */
 463                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 464                         /*
 465                          * Destination field overflow. Copy leftovers of
 466                          * a source field to the next destination field.
 467                          */
 468                         carry_b = 0;
 469                         if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) &&
 470                             dcopy->size != 0) {
 471                                 actions[i].length =
 472                                         dcopy->size * CHAR_BIT - dcopy->offset;
 473                                 carry_b = actions[i].length;
 474                                 next_field = false;
 475                         }
 476                         /*
 477                          * Not enough bits in a source filed to fill a
 478                          * destination field. Switch to the next source.
 479                          */
 480                         if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) &&
 481                             (size_b == field->size * CHAR_BIT - off_b)) {
 482                                 actions[i].length =
 483                                         field->size * CHAR_BIT - off_b;
 484                                 dcopy->offset += actions[i].length;
 485                                 next_dcopy = false;
 486                         }
 487                         if (next_dcopy)
 488                                 ++dcopy;
 489                 } else {
 490                         MLX5_ASSERT(item->spec);
 491                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 492                                                    field->offset, field->size);
 493                         /* Shift out the trailing masked bits from data. */
 494                         data = (data & mask) >> off_b;
 495                         actions[i].data1 = rte_cpu_to_be_32(data);
 496                 }
 497                 /* Convert entire record to expected big-endian format. */
 498                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 499                 if (next_field)
 500                         ++field;
 501                 ++i;
 502         } while (field->size);
 503         if (resource->actions_num == i)
 504                 return rte_flow_error_set(error, EINVAL,
 505                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 506                                           "invalid modification flow item");
 507         resource->actions_num = i;
 508         return 0;
 509 }
 510
 511 /**
 512  * Convert modify-header set IPv4 address action to DV specification.
 513  *
 514  * @param[in,out] resource
 515  *   Pointer to the modify-header resource.
 516  * @param[in] action
 517  *   Pointer to action specification.
 518  * @param[out] error
 519  *   Pointer to the error structure.
 520  *
 521  * @return
 522  *   0 on success, a negative errno value otherwise and rte_errno is set.
 523  */
 524 static int
 525 flow_dv_convert_action_modify_ipv4
 526                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 527                          const struct rte_flow_action *action,
 528                          struct rte_flow_error *error)
 529 {
 530         const struct rte_flow_action_set_ipv4 *conf =
 531                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 532         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 533         struct rte_flow_item_ipv4 ipv4;
 534         struct rte_flow_item_ipv4 ipv4_mask;
 535
 536         memset(&ipv4, 0, sizeof(ipv4));
 537         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 538         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 539                 ipv4.hdr.src_addr = conf->ipv4_addr;
 540                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 541         } else {
 542                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 543                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 544         }
 545         item.spec = &ipv4;
 546         item.mask = &ipv4_mask;
 547         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 548                                              MLX5_MODIFICATION_TYPE_SET, error);
 549 }
 550
 551 /**
 552  * Convert modify-header set IPv6 address action to DV specification.
 553  *
 554  * @param[in,out] resource
 555  *   Pointer to the modify-header resource.
 556  * @param[in] action
 557  *   Pointer to action specification.
 558  * @param[out] error
 559  *   Pointer to the error structure.
 560  *
 561  * @return
 562  *   0 on success, a negative errno value otherwise and rte_errno is set.
 563  */
 564 static int
 565 flow_dv_convert_action_modify_ipv6
 566                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 567                          const struct rte_flow_action *action,
 568                          struct rte_flow_error *error)
 569 {
 570         const struct rte_flow_action_set_ipv6 *conf =
 571                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 572         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 573         struct rte_flow_item_ipv6 ipv6;
 574         struct rte_flow_item_ipv6 ipv6_mask;
 575
 576         memset(&ipv6, 0, sizeof(ipv6));
 577         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 578         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 579                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 580                        sizeof(ipv6.hdr.src_addr));
 581                 memcpy(&ipv6_mask.hdr.src_addr,
 582                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 583                        sizeof(ipv6.hdr.src_addr));
 584         } else {
 585                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 586                        sizeof(ipv6.hdr.dst_addr));
 587                 memcpy(&ipv6_mask.hdr.dst_addr,
 588                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 589                        sizeof(ipv6.hdr.dst_addr));
 590         }
 591         item.spec = &ipv6;
 592         item.mask = &ipv6_mask;
 593         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 594                                              MLX5_MODIFICATION_TYPE_SET, error);
 595 }
 596
 597 /**
 598  * Convert modify-header set MAC address action to DV specification.
 599  *
 600  * @param[in,out] resource
 601  *   Pointer to the modify-header resource.
 602  * @param[in] action
 603  *   Pointer to action specification.
 604  * @param[out] error
 605  *   Pointer to the error structure.
 606  *
 607  * @return
 608  *   0 on success, a negative errno value otherwise and rte_errno is set.
 609  */
 610 static int
 611 flow_dv_convert_action_modify_mac
 612                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 613                          const struct rte_flow_action *action,
 614                          struct rte_flow_error *error)
 615 {
 616         const struct rte_flow_action_set_mac *conf =
 617                 (const struct rte_flow_action_set_mac *)(action->conf);
 618         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 619         struct rte_flow_item_eth eth;
 620         struct rte_flow_item_eth eth_mask;
 621
 622         memset(&eth, 0, sizeof(eth));
 623         memset(&eth_mask, 0, sizeof(eth_mask));
 624         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 625                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 626                        sizeof(eth.src.addr_bytes));
 627                 memcpy(&eth_mask.src.addr_bytes,
 628                        &rte_flow_item_eth_mask.src.addr_bytes,
 629                        sizeof(eth_mask.src.addr_bytes));
 630         } else {
 631                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 632                        sizeof(eth.dst.addr_bytes));
 633                 memcpy(&eth_mask.dst.addr_bytes,
 634                        &rte_flow_item_eth_mask.dst.addr_bytes,
 635                        sizeof(eth_mask.dst.addr_bytes));
 636         }
 637         item.spec = &eth;
 638         item.mask = &eth_mask;
 639         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 640                                              MLX5_MODIFICATION_TYPE_SET, error);
 641 }
 642
 643 /**
 644  * Convert modify-header set VLAN VID action to DV specification.
 645  *
 646  * @param[in,out] resource
 647  *   Pointer to the modify-header resource.
 648  * @param[in] action
 649  *   Pointer to action specification.
 650  * @param[out] error
 651  *   Pointer to the error structure.
 652  *
 653  * @return
 654  *   0 on success, a negative errno value otherwise and rte_errno is set.
 655  */
 656 static int
 657 flow_dv_convert_action_modify_vlan_vid
 658                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 659                          const struct rte_flow_action *action,
 660                          struct rte_flow_error *error)
 661 {
 662         const struct rte_flow_action_of_set_vlan_vid *conf =
 663                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 664         int i = resource->actions_num;
 665         struct mlx5_modification_cmd *actions = resource->actions;
 666         struct field_modify_info *field = modify_vlan_out_first_vid;
 667
 668         if (i >= MLX5_MAX_MODIFY_NUM)
 669                 return rte_flow_error_set(error, EINVAL,
 670                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 671                          "too many items to modify");
 672         actions[i] = (struct mlx5_modification_cmd) {
 673                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 674                 .field = field->id,
 675                 .length = field->size,
 676                 .offset = field->offset,
 677         };
 678         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 679         actions[i].data1 = conf->vlan_vid;
 680         actions[i].data1 = actions[i].data1 << 16;
 681         resource->actions_num = ++i;
 682         return 0;
 683 }
 684
 685 /**
 686  * Convert modify-header set TP action to DV specification.
 687  *
 688  * @param[in,out] resource
 689  *   Pointer to the modify-header resource.
 690  * @param[in] action
 691  *   Pointer to action specification.
 692  * @param[in] items
 693  *   Pointer to rte_flow_item objects list.
 694  * @param[in] attr
 695  *   Pointer to flow attributes structure.
 696  * @param[in] dev_flow
 697  *   Pointer to the sub flow.
 698  * @param[in] tunnel_decap
 699  *   Whether action is after tunnel decapsulation.
 700  * @param[out] error
 701  *   Pointer to the error structure.
 702  *
 703  * @return
 704  *   0 on success, a negative errno value otherwise and rte_errno is set.
 705  */
 706 static int
 707 flow_dv_convert_action_modify_tp
 708                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 709                          const struct rte_flow_action *action,
 710                          const struct rte_flow_item *items,
 711                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 712                          bool tunnel_decap, struct rte_flow_error *error)
 713 {
 714         const struct rte_flow_action_set_tp *conf =
 715                 (const struct rte_flow_action_set_tp *)(action->conf);
 716         struct rte_flow_item item;
 717         struct rte_flow_item_udp udp;
 718         struct rte_flow_item_udp udp_mask;
 719         struct rte_flow_item_tcp tcp;
 720         struct rte_flow_item_tcp tcp_mask;
 721         struct field_modify_info *field;
 722
 723         if (!attr->valid)
 724                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 725         if (attr->udp) {
 726                 memset(&udp, 0, sizeof(udp));
 727                 memset(&udp_mask, 0, sizeof(udp_mask));
 728                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 729                         udp.hdr.src_port = conf->port;
 730                         udp_mask.hdr.src_port =
 731                                         rte_flow_item_udp_mask.hdr.src_port;
 732                 } else {
 733                         udp.hdr.dst_port = conf->port;
 734                         udp_mask.hdr.dst_port =
 735                                         rte_flow_item_udp_mask.hdr.dst_port;
 736                 }
 737                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 738                 item.spec = &udp;
 739                 item.mask = &udp_mask;
 740                 field = modify_udp;
 741         } else {
 742                 MLX5_ASSERT(attr->tcp);
 743                 memset(&tcp, 0, sizeof(tcp));
 744                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 745                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 746                         tcp.hdr.src_port = conf->port;
 747                         tcp_mask.hdr.src_port =
 748                                         rte_flow_item_tcp_mask.hdr.src_port;
 749                 } else {
 750                         tcp.hdr.dst_port = conf->port;
 751                         tcp_mask.hdr.dst_port =
 752                                         rte_flow_item_tcp_mask.hdr.dst_port;
 753                 }
 754                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 755                 item.spec = &tcp;
 756                 item.mask = &tcp_mask;
 757                 field = modify_tcp;
 758         }
 759         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 760                                              MLX5_MODIFICATION_TYPE_SET, error);
 761 }
 762
 763 /**
 764  * Convert modify-header set TTL action to DV specification.
 765  *
 766  * @param[in,out] resource
 767  *   Pointer to the modify-header resource.
 768  * @param[in] action
 769  *   Pointer to action specification.
 770  * @param[in] items
 771  *   Pointer to rte_flow_item objects list.
 772  * @param[in] attr
 773  *   Pointer to flow attributes structure.
 774  * @param[in] dev_flow
 775  *   Pointer to the sub flow.
 776  * @param[in] tunnel_decap
 777  *   Whether action is after tunnel decapsulation.
 778  * @param[out] error
 779  *   Pointer to the error structure.
 780  *
 781  * @return
 782  *   0 on success, a negative errno value otherwise and rte_errno is set.
 783  */
 784 static int
 785 flow_dv_convert_action_modify_ttl
 786                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 787                          const struct rte_flow_action *action,
 788                          const struct rte_flow_item *items,
 789                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 790                          bool tunnel_decap, struct rte_flow_error *error)
 791 {
 792         const struct rte_flow_action_set_ttl *conf =
 793                 (const struct rte_flow_action_set_ttl *)(action->conf);
 794         struct rte_flow_item item;
 795         struct rte_flow_item_ipv4 ipv4;
 796         struct rte_flow_item_ipv4 ipv4_mask;
 797         struct rte_flow_item_ipv6 ipv6;
 798         struct rte_flow_item_ipv6 ipv6_mask;
 799         struct field_modify_info *field;
 800
 801         if (!attr->valid)
 802                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 803         if (attr->ipv4) {
 804                 memset(&ipv4, 0, sizeof(ipv4));
 805                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 806                 ipv4.hdr.time_to_live = conf->ttl_value;
 807                 ipv4_mask.hdr.time_to_live = 0xFF;
 808                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 809                 item.spec = &ipv4;
 810                 item.mask = &ipv4_mask;
 811                 field = modify_ipv4;
 812         } else {
 813                 MLX5_ASSERT(attr->ipv6);
 814                 memset(&ipv6, 0, sizeof(ipv6));
 815                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 816                 ipv6.hdr.hop_limits = conf->ttl_value;
 817                 ipv6_mask.hdr.hop_limits = 0xFF;
 818                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 819                 item.spec = &ipv6;
 820                 item.mask = &ipv6_mask;
 821                 field = modify_ipv6;
 822         }
 823         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 824                                              MLX5_MODIFICATION_TYPE_SET, error);
 825 }
 826
 827 /**
 828  * Convert modify-header decrement TTL action to DV specification.
 829  *
 830  * @param[in,out] resource
 831  *   Pointer to the modify-header resource.
 832  * @param[in] action
 833  *   Pointer to action specification.
 834  * @param[in] items
 835  *   Pointer to rte_flow_item objects list.
 836  * @param[in] attr
 837  *   Pointer to flow attributes structure.
 838  * @param[in] dev_flow
 839  *   Pointer to the sub flow.
 840  * @param[in] tunnel_decap
 841  *   Whether action is after tunnel decapsulation.
 842  * @param[out] error
 843  *   Pointer to the error structure.
 844  *
 845  * @return
 846  *   0 on success, a negative errno value otherwise and rte_errno is set.
 847  */
 848 static int
 849 flow_dv_convert_action_modify_dec_ttl
 850                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 851                          const struct rte_flow_item *items,
 852                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 853                          bool tunnel_decap, struct rte_flow_error *error)
 854 {
 855         struct rte_flow_item item;
 856         struct rte_flow_item_ipv4 ipv4;
 857         struct rte_flow_item_ipv4 ipv4_mask;
 858         struct rte_flow_item_ipv6 ipv6;
 859         struct rte_flow_item_ipv6 ipv6_mask;
 860         struct field_modify_info *field;
 861
 862         if (!attr->valid)
 863                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 864         if (attr->ipv4) {
 865                 memset(&ipv4, 0, sizeof(ipv4));
 866                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 867                 ipv4.hdr.time_to_live = 0xFF;
 868                 ipv4_mask.hdr.time_to_live = 0xFF;
 869                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 870                 item.spec = &ipv4;
 871                 item.mask = &ipv4_mask;
 872                 field = modify_ipv4;
 873         } else {
 874                 MLX5_ASSERT(attr->ipv6);
 875                 memset(&ipv6, 0, sizeof(ipv6));
 876                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 877                 ipv6.hdr.hop_limits = 0xFF;
 878                 ipv6_mask.hdr.hop_limits = 0xFF;
 879                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 880                 item.spec = &ipv6;
 881                 item.mask = &ipv6_mask;
 882                 field = modify_ipv6;
 883         }
 884         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 885                                              MLX5_MODIFICATION_TYPE_ADD, error);
 886 }
 887
 888 /**
 889  * Convert modify-header increment/decrement TCP Sequence number
 890  * to DV specification.
 891  *
 892  * @param[in,out] resource
 893  *   Pointer to the modify-header resource.
 894  * @param[in] action
 895  *   Pointer to action specification.
 896  * @param[out] error
 897  *   Pointer to the error structure.
 898  *
 899  * @return
 900  *   0 on success, a negative errno value otherwise and rte_errno is set.
 901  */
 902 static int
 903 flow_dv_convert_action_modify_tcp_seq
 904                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 905                          const struct rte_flow_action *action,
 906                          struct rte_flow_error *error)
 907 {
 908         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 909         uint64_t value = rte_be_to_cpu_32(*conf);
 910         struct rte_flow_item item;
 911         struct rte_flow_item_tcp tcp;
 912         struct rte_flow_item_tcp tcp_mask;
 913
 914         memset(&tcp, 0, sizeof(tcp));
 915         memset(&tcp_mask, 0, sizeof(tcp_mask));
 916         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 917                 /*
 918                  * The HW has no decrement operation, only increment operation.
 919                  * To simulate decrement X from Y using increment operation
 920                  * we need to add UINT32_MAX X times to Y.
 921                  * Each adding of UINT32_MAX decrements Y by 1.
 922                  */
 923                 value *= UINT32_MAX;
 924         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 925         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 926         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 927         item.spec = &tcp;
 928         item.mask = &tcp_mask;
 929         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 930                                              MLX5_MODIFICATION_TYPE_ADD, error);
 931 }
 932
 933 /**
 934  * Convert modify-header increment/decrement TCP Acknowledgment number
 935  * to DV specification.
 936  *
 937  * @param[in,out] resource
 938  *   Pointer to the modify-header resource.
 939  * @param[in] action
 940  *   Pointer to action specification.
 941  * @param[out] error
 942  *   Pointer to the error structure.
 943  *
 944  * @return
 945  *   0 on success, a negative errno value otherwise and rte_errno is set.
 946  */
 947 static int
 948 flow_dv_convert_action_modify_tcp_ack
 949                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 950                          const struct rte_flow_action *action,
 951                          struct rte_flow_error *error)
 952 {
 953         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 954         uint64_t value = rte_be_to_cpu_32(*conf);
 955         struct rte_flow_item item;
 956         struct rte_flow_item_tcp tcp;
 957         struct rte_flow_item_tcp tcp_mask;
 958
 959         memset(&tcp, 0, sizeof(tcp));
 960         memset(&tcp_mask, 0, sizeof(tcp_mask));
 961         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 962                 /*
 963                  * The HW has no decrement operation, only increment operation.
 964                  * To simulate decrement X from Y using increment operation
 965                  * we need to add UINT32_MAX X times to Y.
 966                  * Each adding of UINT32_MAX decrements Y by 1.
 967                  */
 968                 value *= UINT32_MAX;
 969         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 970         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 971         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 972         item.spec = &tcp;
 973         item.mask = &tcp_mask;
 974         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 975                                              MLX5_MODIFICATION_TYPE_ADD, error);
 976 }
 977
 978 static enum mlx5_modification_field reg_to_field[] = {
 979         [REG_NON] = MLX5_MODI_OUT_NONE,
 980         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 981         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 982         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 983         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 984         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 985         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 986         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 987         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 988         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 989         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 990 };
 991
 992 /**
 993  * Convert register set to DV specification.
 994  *
 995  * @param[in,out] resource
 996  *   Pointer to the modify-header resource.
 997  * @param[in] action
 998  *   Pointer to action specification.
 999  * @param[out] error
1000  *   Pointer to the error structure.
1001  *
1002  * @return
1003  *   0 on success, a negative errno value otherwise and rte_errno is set.
1004  */
1005 static int
1006 flow_dv_convert_action_set_reg
1007                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1008                          const struct rte_flow_action *action,
1009                          struct rte_flow_error *error)
1010 {
1011         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
1012         struct mlx5_modification_cmd *actions = resource->actions;
1013         uint32_t i = resource->actions_num;
1014
1015         if (i >= MLX5_MAX_MODIFY_NUM)
1016                 return rte_flow_error_set(error, EINVAL,
1017                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1018                                           "too many items to modify");
1019         MLX5_ASSERT(conf->id != REG_NON);
1020         MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
1021         actions[i] = (struct mlx5_modification_cmd) {
1022                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1023                 .field = reg_to_field[conf->id],
1024                 .offset = conf->offset,
1025                 .length = conf->length,
1026         };
1027         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1028         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1029         ++i;
1030         resource->actions_num = i;
1031         return 0;
1032 }
1033
1034 /**
1035  * Convert SET_TAG action to DV specification.
1036  *
1037  * @param[in] dev
1038  *   Pointer to the rte_eth_dev structure.
1039  * @param[in,out] resource
1040  *   Pointer to the modify-header resource.
1041  * @param[in] conf
1042  *   Pointer to action specification.
1043  * @param[out] error
1044  *   Pointer to the error structure.
1045  *
1046  * @return
1047  *   0 on success, a negative errno value otherwise and rte_errno is set.
1048  */
1049 static int
1050 flow_dv_convert_action_set_tag
1051                         (struct rte_eth_dev *dev,
1052                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1053                          const struct rte_flow_action_set_tag *conf,
1054                          struct rte_flow_error *error)
1055 {
1056         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1057         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1058         struct rte_flow_item item = {
1059                 .spec = &data,
1060                 .mask = &mask,
1061         };
1062         struct field_modify_info reg_c_x[] = {
1063                 [1] = {0, 0, 0},
1064         };
1065         enum mlx5_modification_field reg_type;
1066         int ret;
1067
1068         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1069         if (ret < 0)
1070                 return ret;
1071         MLX5_ASSERT(ret != REG_NON);
1072         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1073         reg_type = reg_to_field[ret];
1074         MLX5_ASSERT(reg_type > 0);
1075         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1076         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1077                                              MLX5_MODIFICATION_TYPE_SET, error);
1078 }
1079
1080 /**
1081  * Convert internal COPY_REG action to DV specification.
1082  *
1083  * @param[in] dev
1084  *   Pointer to the rte_eth_dev structure.
1085  * @param[in,out] res
1086  *   Pointer to the modify-header resource.
1087  * @param[in] action
1088  *   Pointer to action specification.
1089  * @param[out] error
1090  *   Pointer to the error structure.
1091  *
1092  * @return
1093  *   0 on success, a negative errno value otherwise and rte_errno is set.
1094  */
1095 static int
1096 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1097                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1098                                  const struct rte_flow_action *action,
1099                                  struct rte_flow_error *error)
1100 {
1101         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1102         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1103         struct rte_flow_item item = {
1104                 .spec = NULL,
1105                 .mask = &mask,
1106         };
1107         struct field_modify_info reg_src[] = {
1108                 {4, 0, reg_to_field[conf->src]},
1109                 {0, 0, 0},
1110         };
1111         struct field_modify_info reg_dst = {
1112                 .offset = 0,
1113                 .id = reg_to_field[conf->dst],
1114         };
1115         /* Adjust reg_c[0] usage according to reported mask. */
1116         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1117                 struct mlx5_priv *priv = dev->data->dev_private;
1118                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1119
1120                 MLX5_ASSERT(reg_c0);
1121                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1122                 if (conf->dst == REG_C_0) {
1123                         /* Copy to reg_c[0], within mask only. */
1124                         reg_dst.offset = rte_bsf32(reg_c0);
1125                         /*
1126                          * Mask is ignoring the enianness, because
1127                          * there is no conversion in datapath.
1128                          */
1129 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1130                         /* Copy from destination lower bits to reg_c[0]. */
1131                         mask = reg_c0 >> reg_dst.offset;
1132 #else
1133                         /* Copy from destination upper bits to reg_c[0]. */
1134                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1135                                           rte_fls_u32(reg_c0));
1136 #endif
1137                 } else {
1138                         mask = rte_cpu_to_be_32(reg_c0);
1139 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1140                         /* Copy from reg_c[0] to destination lower bits. */
1141                         reg_dst.offset = 0;
1142 #else
1143                         /* Copy from reg_c[0] to destination upper bits. */
1144                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1145                                          (rte_fls_u32(reg_c0) -
1146                                           rte_bsf32(reg_c0));
1147 #endif
1148                 }
1149         }
1150         return flow_dv_convert_modify_action(&item,
1151                                              reg_src, &reg_dst, res,
1152                                              MLX5_MODIFICATION_TYPE_COPY,
1153                                              error);
1154 }
1155
1156 /**
1157  * Convert MARK action to DV specification. This routine is used
1158  * in extensive metadata only and requires metadata register to be
1159  * handled. In legacy mode hardware tag resource is engaged.
1160  *
1161  * @param[in] dev
1162  *   Pointer to the rte_eth_dev structure.
1163  * @param[in] conf
1164  *   Pointer to MARK action specification.
1165  * @param[in,out] resource
1166  *   Pointer to the modify-header resource.
1167  * @param[out] error
1168  *   Pointer to the error structure.
1169  *
1170  * @return
1171  *   0 on success, a negative errno value otherwise and rte_errno is set.
1172  */
1173 static int
1174 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1175                             const struct rte_flow_action_mark *conf,
1176                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1177                             struct rte_flow_error *error)
1178 {
1179         struct mlx5_priv *priv = dev->data->dev_private;
1180         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1181                                            priv->sh->dv_mark_mask);
1182         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1183         struct rte_flow_item item = {
1184                 .spec = &data,
1185                 .mask = &mask,
1186         };
1187         struct field_modify_info reg_c_x[] = {
1188                 [1] = {0, 0, 0},
1189         };
1190         int reg;
1191
1192         if (!mask)
1193                 return rte_flow_error_set(error, EINVAL,
1194                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1195                                           NULL, "zero mark action mask");
1196         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1197         if (reg < 0)
1198                 return reg;
1199         MLX5_ASSERT(reg > 0);
1200         if (reg == REG_C_0) {
1201                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1202                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1203
1204                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1205                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1206                 mask = rte_cpu_to_be_32(mask << shl_c0);
1207         }
1208         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1209         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1210                                              MLX5_MODIFICATION_TYPE_SET, error);
1211 }
1212
1213 /**
1214  * Get metadata register index for specified steering domain.
1215  *
1216  * @param[in] dev
1217  *   Pointer to the rte_eth_dev structure.
1218  * @param[in] attr
1219  *   Attributes of flow to determine steering domain.
1220  * @param[out] error
1221  *   Pointer to the error structure.
1222  *
1223  * @return
1224  *   positive index on success, a negative errno value otherwise
1225  *   and rte_errno is set.
1226  */
1227 static enum modify_reg
1228 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1229                          const struct rte_flow_attr *attr,
1230                          struct rte_flow_error *error)
1231 {
1232         int reg =
1233                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1234                                           MLX5_METADATA_FDB :
1235                                             attr->egress ?
1236                                             MLX5_METADATA_TX :
1237                                             MLX5_METADATA_RX, 0, error);
1238         if (reg < 0)
1239                 return rte_flow_error_set(error,
1240                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1241                                           NULL, "unavailable "
1242                                           "metadata register");
1243         return reg;
1244 }
1245
1246 /**
1247  * Convert SET_META action to DV specification.
1248  *
1249  * @param[in] dev
1250  *   Pointer to the rte_eth_dev structure.
1251  * @param[in,out] resource
1252  *   Pointer to the modify-header resource.
1253  * @param[in] attr
1254  *   Attributes of flow that includes this item.
1255  * @param[in] conf
1256  *   Pointer to action specification.
1257  * @param[out] error
1258  *   Pointer to the error structure.
1259  *
1260  * @return
1261  *   0 on success, a negative errno value otherwise and rte_errno is set.
1262  */
1263 static int
1264 flow_dv_convert_action_set_meta
1265                         (struct rte_eth_dev *dev,
1266                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1267                          const struct rte_flow_attr *attr,
1268                          const struct rte_flow_action_set_meta *conf,
1269                          struct rte_flow_error *error)
1270 {
1271         uint32_t mask = rte_cpu_to_be_32(conf->mask);
1272         uint32_t data = rte_cpu_to_be_32(conf->data) & mask;
1273         struct rte_flow_item item = {
1274                 .spec = &data,
1275                 .mask = &mask,
1276         };
1277         struct field_modify_info reg_c_x[] = {
1278                 [1] = {0, 0, 0},
1279         };
1280         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1281
1282         if (reg < 0)
1283                 return reg;
1284         MLX5_ASSERT(reg != REG_NON);
1285         if (reg == REG_C_0) {
1286                 struct mlx5_priv *priv = dev->data->dev_private;
1287                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1288                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1289
1290                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1291                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1292                 mask = rte_cpu_to_be_32(mask << shl_c0);
1293         }
1294         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1295         /* The routine expects parameters in memory as big-endian ones. */
1296         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1297                                              MLX5_MODIFICATION_TYPE_SET, error);
1298 }
1299
1300 /**
1301  * Convert modify-header set IPv4 DSCP action to DV specification.
1302  *
1303  * @param[in,out] resource
1304  *   Pointer to the modify-header resource.
1305  * @param[in] action
1306  *   Pointer to action specification.
1307  * @param[out] error
1308  *   Pointer to the error structure.
1309  *
1310  * @return
1311  *   0 on success, a negative errno value otherwise and rte_errno is set.
1312  */
1313 static int
1314 flow_dv_convert_action_modify_ipv4_dscp
1315                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1316                          const struct rte_flow_action *action,
1317                          struct rte_flow_error *error)
1318 {
1319         const struct rte_flow_action_set_dscp *conf =
1320                 (const struct rte_flow_action_set_dscp *)(action->conf);
1321         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1322         struct rte_flow_item_ipv4 ipv4;
1323         struct rte_flow_item_ipv4 ipv4_mask;
1324
1325         memset(&ipv4, 0, sizeof(ipv4));
1326         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1327         ipv4.hdr.type_of_service = conf->dscp;
1328         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1329         item.spec = &ipv4;
1330         item.mask = &ipv4_mask;
1331         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1332                                              MLX5_MODIFICATION_TYPE_SET, error);
1333 }
1334
1335 /**
1336  * Convert modify-header set IPv6 DSCP action to DV specification.
1337  *
1338  * @param[in,out] resource
1339  *   Pointer to the modify-header resource.
1340  * @param[in] action
1341  *   Pointer to action specification.
1342  * @param[out] error
1343  *   Pointer to the error structure.
1344  *
1345  * @return
1346  *   0 on success, a negative errno value otherwise and rte_errno is set.
1347  */
1348 static int
1349 flow_dv_convert_action_modify_ipv6_dscp
1350                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1351                          const struct rte_flow_action *action,
1352                          struct rte_flow_error *error)
1353 {
1354         const struct rte_flow_action_set_dscp *conf =
1355                 (const struct rte_flow_action_set_dscp *)(action->conf);
1356         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1357         struct rte_flow_item_ipv6 ipv6;
1358         struct rte_flow_item_ipv6 ipv6_mask;
1359
1360         memset(&ipv6, 0, sizeof(ipv6));
1361         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1362         /*
1363          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1364          * rdma-core only accept the DSCP bits byte aligned start from
1365          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1366          * bits in IPv6 case as rdma-core requires byte aligned value.
1367          */
1368         ipv6.hdr.vtc_flow = conf->dscp;
1369         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1370         item.spec = &ipv6;
1371         item.mask = &ipv6_mask;
1372         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1373                                              MLX5_MODIFICATION_TYPE_SET, error);
1374 }
1375
1376 static int
1377 mlx5_flow_item_field_width(struct mlx5_dev_config *config,
1378                            enum rte_flow_field_id field)
1379 {
1380         switch (field) {
1381         case RTE_FLOW_FIELD_START:
1382                 return 32;
1383         case RTE_FLOW_FIELD_MAC_DST:
1384         case RTE_FLOW_FIELD_MAC_SRC:
1385                 return 48;
1386         case RTE_FLOW_FIELD_VLAN_TYPE:
1387                 return 16;
1388         case RTE_FLOW_FIELD_VLAN_ID:
1389                 return 12;
1390         case RTE_FLOW_FIELD_MAC_TYPE:
1391                 return 16;
1392         case RTE_FLOW_FIELD_IPV4_DSCP:
1393                 return 6;
1394         case RTE_FLOW_FIELD_IPV4_TTL:
1395                 return 8;
1396         case RTE_FLOW_FIELD_IPV4_SRC:
1397         case RTE_FLOW_FIELD_IPV4_DST:
1398                 return 32;
1399         case RTE_FLOW_FIELD_IPV6_DSCP:
1400                 return 6;
1401         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1402                 return 8;
1403         case RTE_FLOW_FIELD_IPV6_SRC:
1404         case RTE_FLOW_FIELD_IPV6_DST:
1405                 return 128;
1406         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1407         case RTE_FLOW_FIELD_TCP_PORT_DST:
1408                 return 16;
1409         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1410         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1411                 return 32;
1412         case RTE_FLOW_FIELD_TCP_FLAGS:
1413                 return 9;
1414         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1415         case RTE_FLOW_FIELD_UDP_PORT_DST:
1416                 return 16;
1417         case RTE_FLOW_FIELD_VXLAN_VNI:
1418         case RTE_FLOW_FIELD_GENEVE_VNI:
1419                 return 24;
1420         case RTE_FLOW_FIELD_GTP_TEID:
1421         case RTE_FLOW_FIELD_TAG:
1422                 return 32;
1423         case RTE_FLOW_FIELD_MARK:
1424                 return 24;
1425         case RTE_FLOW_FIELD_META:
1426                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_META16)
1427                         return 16;
1428                 else if (config->dv_xmeta_en == MLX5_XMETA_MODE_META32)
1429                         return 32;
1430                 else
1431                         return 0;
1432         case RTE_FLOW_FIELD_POINTER:
1433         case RTE_FLOW_FIELD_VALUE:
1434                 return 64;
1435         default:
1436                 MLX5_ASSERT(false);
1437         }
1438         return 0;
1439 }
1440
1441 static void
1442 mlx5_flow_field_id_to_modify_info
1443                 (const struct rte_flow_action_modify_data *data,
1444                  struct field_modify_info *info,
1445                  uint32_t *mask, uint32_t *value,
1446                  uint32_t width, uint32_t dst_width,
1447                  struct rte_eth_dev *dev,
1448                  const struct rte_flow_attr *attr,
1449                  struct rte_flow_error *error)
1450 {
1451         struct mlx5_priv *priv = dev->data->dev_private;
1452         struct mlx5_dev_config *config = &priv->config;
1453         uint32_t idx = 0;
1454         uint32_t off = 0;
1455         uint64_t val = 0;
1456         switch (data->field) {
1457         case RTE_FLOW_FIELD_START:
1458                 /* not supported yet */
1459                 MLX5_ASSERT(false);
1460                 break;
1461         case RTE_FLOW_FIELD_MAC_DST:
1462                 off = data->offset > 16 ? data->offset - 16 : 0;
1463                 if (mask) {
1464                         if (data->offset < 16) {
1465                                 info[idx] = (struct field_modify_info){2, 0,
1466                                                 MLX5_MODI_OUT_DMAC_15_0};
1467                                 if (width < 16) {
1468                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1469                                                                  (16 - width));
1470                                         width = 0;
1471                                 } else {
1472                                         mask[idx] = RTE_BE16(0xffff);
1473                                         width -= 16;
1474                                 }
1475                                 if (!width)
1476                                         break;
1477                                 ++idx;
1478                         }
1479                         info[idx] = (struct field_modify_info){4, 4 * idx,
1480                                                 MLX5_MODI_OUT_DMAC_47_16};
1481                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1482                                                       (32 - width)) << off);
1483                 } else {
1484                         if (data->offset < 16)
1485                                 info[idx++] = (struct field_modify_info){2, 0,
1486                                                 MLX5_MODI_OUT_DMAC_15_0};
1487                         info[idx] = (struct field_modify_info){4, off,
1488                                                 MLX5_MODI_OUT_DMAC_47_16};
1489                 }
1490                 break;
1491         case RTE_FLOW_FIELD_MAC_SRC:
1492                 off = data->offset > 16 ? data->offset - 16 : 0;
1493                 if (mask) {
1494                         if (data->offset < 16) {
1495                                 info[idx] = (struct field_modify_info){2, 0,
1496                                                 MLX5_MODI_OUT_SMAC_15_0};
1497                                 if (width < 16) {
1498                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1499                                                                  (16 - width));
1500                                         width = 0;
1501                                 } else {
1502                                         mask[idx] = RTE_BE16(0xffff);
1503                                         width -= 16;
1504                                 }
1505                                 if (!width)
1506                                         break;
1507                                 ++idx;
1508                         }
1509                         info[idx] = (struct field_modify_info){4, 4 * idx,
1510                                                 MLX5_MODI_OUT_SMAC_47_16};
1511                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1512                                                       (32 - width)) << off);
1513                 } else {
1514                         if (data->offset < 16)
1515                                 info[idx++] = (struct field_modify_info){2, 0,
1516                                                 MLX5_MODI_OUT_SMAC_15_0};
1517                         info[idx] = (struct field_modify_info){4, off,
1518                                                 MLX5_MODI_OUT_SMAC_47_16};
1519                 }
1520                 break;
1521         case RTE_FLOW_FIELD_VLAN_TYPE:
1522                 /* not supported yet */
1523                 break;
1524         case RTE_FLOW_FIELD_VLAN_ID:
1525                 info[idx] = (struct field_modify_info){2, 0,
1526                                         MLX5_MODI_OUT_FIRST_VID};
1527                 if (mask)
1528                         mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
1529                 break;
1530         case RTE_FLOW_FIELD_MAC_TYPE:
1531                 info[idx] = (struct field_modify_info){2, 0,
1532                                         MLX5_MODI_OUT_ETHERTYPE};
1533                 if (mask)
1534                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1535                 break;
1536         case RTE_FLOW_FIELD_IPV4_DSCP:
1537                 info[idx] = (struct field_modify_info){1, 0,
1538                                         MLX5_MODI_OUT_IP_DSCP};
1539                 if (mask)
1540                         mask[idx] = 0x3f >> (6 - width);
1541                 break;
1542         case RTE_FLOW_FIELD_IPV4_TTL:
1543                 info[idx] = (struct field_modify_info){1, 0,
1544                                         MLX5_MODI_OUT_IPV4_TTL};
1545                 if (mask)
1546                         mask[idx] = 0xff >> (8 - width);
1547                 break;
1548         case RTE_FLOW_FIELD_IPV4_SRC:
1549                 info[idx] = (struct field_modify_info){4, 0,
1550                                         MLX5_MODI_OUT_SIPV4};
1551                 if (mask)
1552                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1553                                                      (32 - width));
1554                 break;
1555         case RTE_FLOW_FIELD_IPV4_DST:
1556                 info[idx] = (struct field_modify_info){4, 0,
1557                                         MLX5_MODI_OUT_DIPV4};
1558                 if (mask)
1559                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1560                                                      (32 - width));
1561                 break;
1562         case RTE_FLOW_FIELD_IPV6_DSCP:
1563                 info[idx] = (struct field_modify_info){1, 0,
1564                                         MLX5_MODI_OUT_IP_DSCP};
1565                 if (mask)
1566                         mask[idx] = 0x3f >> (6 - width);
1567                 break;
1568         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1569                 info[idx] = (struct field_modify_info){1, 0,
1570                                         MLX5_MODI_OUT_IPV6_HOPLIMIT};
1571                 if (mask)
1572                         mask[idx] = 0xff >> (8 - width);
1573                 break;
1574         case RTE_FLOW_FIELD_IPV6_SRC:
1575                 if (mask) {
1576                         if (data->offset < 32) {
1577                                 info[idx] = (struct field_modify_info){4,
1578                                                 4 * idx,
1579                                                 MLX5_MODI_OUT_SIPV6_31_0};
1580                                 if (width < 32) {
1581                                         mask[idx] =
1582                                                 rte_cpu_to_be_32(0xffffffff >>
1583                                                                  (32 - width));
1584                                         width = 0;
1585                                 } else {
1586                                         mask[idx] = RTE_BE32(0xffffffff);
1587                                         width -= 32;
1588                                 }
1589                                 if (!width)
1590                                         break;
1591                                 ++idx;
1592                         }
1593                         if (data->offset < 64) {
1594                                 info[idx] = (struct field_modify_info){4,
1595                                                 4 * idx,
1596                                                 MLX5_MODI_OUT_SIPV6_63_32};
1597                                 if (width < 32) {
1598                                         mask[idx] =
1599                                                 rte_cpu_to_be_32(0xffffffff >>
1600                                                                  (32 - width));
1601                                         width = 0;
1602                                 } else {
1603                                         mask[idx] = RTE_BE32(0xffffffff);
1604                                         width -= 32;
1605                                 }
1606                                 if (!width)
1607                                         break;
1608                                 ++idx;
1609                         }
1610                         if (data->offset < 96) {
1611                                 info[idx] = (struct field_modify_info){4,
1612                                                 4 * idx,
1613                                                 MLX5_MODI_OUT_SIPV6_95_64};
1614                                 if (width < 32) {
1615                                         mask[idx] =
1616                                                 rte_cpu_to_be_32(0xffffffff >>
1617                                                                  (32 - width));
1618                                         width = 0;
1619                                 } else {
1620                                         mask[idx] = RTE_BE32(0xffffffff);
1621                                         width -= 32;
1622                                 }
1623                                 if (!width)
1624                                         break;
1625                                 ++idx;
1626                         }
1627                         info[idx] = (struct field_modify_info){4, 4 * idx,
1628                                                 MLX5_MODI_OUT_SIPV6_127_96};
1629                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1630                                                      (32 - width));
1631                 } else {
1632                         if (data->offset < 32)
1633                                 info[idx++] = (struct field_modify_info){4, 0,
1634                                                 MLX5_MODI_OUT_SIPV6_31_0};
1635                         if (data->offset < 64)
1636                                 info[idx++] = (struct field_modify_info){4, 0,
1637                                                 MLX5_MODI_OUT_SIPV6_63_32};
1638                         if (data->offset < 96)
1639                                 info[idx++] = (struct field_modify_info){4, 0,
1640                                                 MLX5_MODI_OUT_SIPV6_95_64};
1641                         if (data->offset < 128)
1642                                 info[idx++] = (struct field_modify_info){4, 0,
1643                                                 MLX5_MODI_OUT_SIPV6_127_96};
1644                 }
1645                 break;
1646         case RTE_FLOW_FIELD_IPV6_DST:
1647                 if (mask) {
1648                         if (data->offset < 32) {
1649                                 info[idx] = (struct field_modify_info){4,
1650                                                 4 * idx,
1651                                                 MLX5_MODI_OUT_DIPV6_31_0};
1652                                 if (width < 32) {
1653                                         mask[idx] =
1654                                                 rte_cpu_to_be_32(0xffffffff >>
1655                                                                  (32 - width));
1656                                         width = 0;
1657                                 } else {
1658                                         mask[idx] = RTE_BE32(0xffffffff);
1659                                         width -= 32;
1660                                 }
1661                                 if (!width)
1662                                         break;
1663                                 ++idx;
1664                         }
1665                         if (data->offset < 64) {
1666                                 info[idx] = (struct field_modify_info){4,
1667                                                 4 * idx,
1668                                                 MLX5_MODI_OUT_DIPV6_63_32};
1669                                 if (width < 32) {
1670                                         mask[idx] =
1671                                                 rte_cpu_to_be_32(0xffffffff >>
1672                                                                  (32 - width));
1673                                         width = 0;
1674                                 } else {
1675                                         mask[idx] = RTE_BE32(0xffffffff);
1676                                         width -= 32;
1677                                 }
1678                                 if (!width)
1679                                         break;
1680                                 ++idx;
1681                         }
1682                         if (data->offset < 96) {
1683                                 info[idx] = (struct field_modify_info){4,
1684                                                 4 * idx,
1685                                                 MLX5_MODI_OUT_DIPV6_95_64};
1686                                 if (width < 32) {
1687                                         mask[idx] =
1688                                                 rte_cpu_to_be_32(0xffffffff >>
1689                                                                  (32 - width));
1690                                         width = 0;
1691                                 } else {
1692                                         mask[idx] = RTE_BE32(0xffffffff);
1693                                         width -= 32;
1694                                 }
1695                                 if (!width)
1696                                         break;
1697                                 ++idx;
1698                         }
1699                         info[idx] = (struct field_modify_info){4, 4 * idx,
1700                                                 MLX5_MODI_OUT_DIPV6_127_96};
1701                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1702                                                      (32 - width));
1703                 } else {
1704                         if (data->offset < 32)
1705                                 info[idx++] = (struct field_modify_info){4, 0,
1706                                                 MLX5_MODI_OUT_DIPV6_31_0};
1707                         if (data->offset < 64)
1708                                 info[idx++] = (struct field_modify_info){4, 0,
1709                                                 MLX5_MODI_OUT_DIPV6_63_32};
1710                         if (data->offset < 96)
1711                                 info[idx++] = (struct field_modify_info){4, 0,
1712                                                 MLX5_MODI_OUT_DIPV6_95_64};
1713                         if (data->offset < 128)
1714                                 info[idx++] = (struct field_modify_info){4, 0,
1715                                                 MLX5_MODI_OUT_DIPV6_127_96};
1716                 }
1717                 break;
1718         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1719                 info[idx] = (struct field_modify_info){2, 0,
1720                                         MLX5_MODI_OUT_TCP_SPORT};
1721                 if (mask)
1722                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1723                 break;
1724         case RTE_FLOW_FIELD_TCP_PORT_DST:
1725                 info[idx] = (struct field_modify_info){2, 0,
1726                                         MLX5_MODI_OUT_TCP_DPORT};
1727                 if (mask)
1728                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1729                 break;
1730         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1731                 info[idx] = (struct field_modify_info){4, 0,
1732                                         MLX5_MODI_OUT_TCP_SEQ_NUM};
1733                 if (mask)
1734                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1735                                                      (32 - width));
1736                 break;
1737         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1738                 info[idx] = (struct field_modify_info){4, 0,
1739                                         MLX5_MODI_OUT_TCP_ACK_NUM};
1740                 if (mask)
1741                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1742                                                      (32 - width));
1743                 break;
1744         case RTE_FLOW_FIELD_TCP_FLAGS:
1745                 info[idx] = (struct field_modify_info){2, 0,
1746                                         MLX5_MODI_OUT_TCP_FLAGS};
1747                 if (mask)
1748                         mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
1749                 break;
1750         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1751                 info[idx] = (struct field_modify_info){2, 0,
1752                                         MLX5_MODI_OUT_UDP_SPORT};
1753                 if (mask)
1754                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1755                 break;
1756         case RTE_FLOW_FIELD_UDP_PORT_DST:
1757                 info[idx] = (struct field_modify_info){2, 0,
1758                                         MLX5_MODI_OUT_UDP_DPORT};
1759                 if (mask)
1760                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1761                 break;
1762         case RTE_FLOW_FIELD_VXLAN_VNI:
1763                 /* not supported yet */
1764                 break;
1765         case RTE_FLOW_FIELD_GENEVE_VNI:
1766                 /* not supported yet*/
1767                 break;
1768         case RTE_FLOW_FIELD_GTP_TEID:
1769                 info[idx] = (struct field_modify_info){4, 0,
1770                                         MLX5_MODI_GTP_TEID};
1771                 if (mask)
1772                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1773                                                      (32 - width));
1774                 break;
1775         case RTE_FLOW_FIELD_TAG:
1776                 {
1777                         int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
1778                                                    data->level, error);
1779                         if (reg < 0)
1780                                 return;
1781                         MLX5_ASSERT(reg != REG_NON);
1782                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1783                         info[idx] = (struct field_modify_info){4, 0,
1784                                                 reg_to_field[reg]};
1785                         if (mask)
1786                                 mask[idx] =
1787                                         rte_cpu_to_be_32(0xffffffff >>
1788                                                          (32 - width));
1789                 }
1790                 break;
1791         case RTE_FLOW_FIELD_MARK:
1792                 {
1793                         int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
1794                                                        0, error);
1795                         if (reg < 0)
1796                                 return;
1797                         MLX5_ASSERT(reg != REG_NON);
1798                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1799                         info[idx] = (struct field_modify_info){4, 0,
1800                                                 reg_to_field[reg]};
1801                         if (mask)
1802                                 mask[idx] =
1803                                         rte_cpu_to_be_32(0xffffffff >>
1804                                                          (32 - width));
1805                 }
1806                 break;
1807         case RTE_FLOW_FIELD_META:
1808                 {
1809                         unsigned int xmeta = config->dv_xmeta_en;
1810                         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1811                         if (reg < 0)
1812                                 return;
1813                         MLX5_ASSERT(reg != REG_NON);
1814                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1815                         if (xmeta == MLX5_XMETA_MODE_META16) {
1816                                 info[idx] = (struct field_modify_info){2, 0,
1817                                                         reg_to_field[reg]};
1818                                 if (mask)
1819                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1820                                                                 (16 - width));
1821                         } else if (xmeta == MLX5_XMETA_MODE_META32) {
1822                                 info[idx] = (struct field_modify_info){4, 0,
1823                                                         reg_to_field[reg]};
1824                                 if (mask)
1825                                         mask[idx] =
1826                                                 rte_cpu_to_be_32(0xffffffff >>
1827                                                                 (32 - width));
1828                         } else {
1829                                 MLX5_ASSERT(false);
1830                         }
1831                 }
1832                 break;
1833         case RTE_FLOW_FIELD_POINTER:
1834         case RTE_FLOW_FIELD_VALUE:
1835                 if (data->field == RTE_FLOW_FIELD_POINTER)
1836                         memcpy(&val, (void *)(uintptr_t)data->value,
1837                                sizeof(uint64_t));
1838                 else
1839                         val = data->value;
1840                 for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) {
1841                         if (mask[idx]) {
1842                                 if (dst_width == 48) {
1843                                         /*special case for MAC addresses */
1844                                         value[idx] = rte_cpu_to_be_16(val);
1845                                         val >>= 16;
1846                                         dst_width -= 16;
1847                                 } else if (dst_width > 16) {
1848                                         value[idx] = rte_cpu_to_be_32(val);
1849                                         val >>= 32;
1850                                 } else if (dst_width > 8) {
1851                                         value[idx] = rte_cpu_to_be_16(val);
1852                                         val >>= 16;
1853                                 } else {
1854                                         value[idx] = (uint8_t)val;
1855                                         val >>= 8;
1856                                 }
1857                                 if (!val)
1858                                         break;
1859                         }
1860                 }
1861                 break;
1862         default:
1863                 MLX5_ASSERT(false);
1864                 break;
1865         }
1866 }
1867
1868 /**
1869  * Convert modify_field action to DV specification.
1870  *
1871  * @param[in] dev
1872  *   Pointer to the rte_eth_dev structure.
1873  * @param[in,out] resource
1874  *   Pointer to the modify-header resource.
1875  * @param[in] action
1876  *   Pointer to action specification.
1877  * @param[in] attr
1878  *   Attributes of flow that includes this item.
1879  * @param[out] error
1880  *   Pointer to the error structure.
1881  *
1882  * @return
1883  *   0 on success, a negative errno value otherwise and rte_errno is set.
1884  */
1885 static int
1886 flow_dv_convert_action_modify_field
1887                         (struct rte_eth_dev *dev,
1888                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1889                          const struct rte_flow_action *action,
1890                          const struct rte_flow_attr *attr,
1891                          struct rte_flow_error *error)
1892 {
1893         struct mlx5_priv *priv = dev->data->dev_private;
1894         struct mlx5_dev_config *config = &priv->config;
1895         const struct rte_flow_action_modify_field *conf =
1896                 (const struct rte_flow_action_modify_field *)(action->conf);
1897         struct rte_flow_item item;
1898         struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
1899                                                                 {0, 0, 0} };
1900         struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
1901                                                                 {0, 0, 0} };
1902         uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1903         uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1904         uint32_t type;
1905         uint32_t dst_width = mlx5_flow_item_field_width(config,
1906                                                         conf->dst.field);
1907
1908         if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
1909                 conf->src.field == RTE_FLOW_FIELD_VALUE) {
1910                 type = MLX5_MODIFICATION_TYPE_SET;
1911                 /** For SET fill the destination field (field) first. */
1912                 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
1913                         value, conf->width, dst_width, dev, attr, error);
1914                 /** Then copy immediate value from source as per mask. */
1915                 mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask,
1916                         value, conf->width, dst_width, dev, attr, error);
1917                 item.spec = &value;
1918         } else {
1919                 type = MLX5_MODIFICATION_TYPE_COPY;
1920                 /** For COPY fill the destination field (dcopy) without mask. */
1921                 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
1922                         value, conf->width, dst_width, dev, attr, error);
1923                 /** Then construct the source field (field) with mask. */
1924                 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
1925                         value, conf->width, dst_width, dev, attr, error);
1926         }
1927         item.mask = &mask;
1928         return flow_dv_convert_modify_action(&item,
1929                         field, dcopy, resource, type, error);
1930 }
1931
1932 /**
1933  * Validate MARK item.
1934  *
1935  * @param[in] dev
1936  *   Pointer to the rte_eth_dev structure.
1937  * @param[in] item
1938  *   Item specification.
1939  * @param[in] attr
1940  *   Attributes of flow that includes this item.
1941  * @param[out] error
1942  *   Pointer to error structure.
1943  *
1944  * @return
1945  *   0 on success, a negative errno value otherwise and rte_errno is set.
1946  */
1947 static int
1948 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1949                            const struct rte_flow_item *item,
1950                            const struct rte_flow_attr *attr __rte_unused,
1951                            struct rte_flow_error *error)
1952 {
1953         struct mlx5_priv *priv = dev->data->dev_private;
1954         struct mlx5_dev_config *config = &priv->config;
1955         const struct rte_flow_item_mark *spec = item->spec;
1956         const struct rte_flow_item_mark *mask = item->mask;
1957         const struct rte_flow_item_mark nic_mask = {
1958                 .id = priv->sh->dv_mark_mask,
1959         };
1960         int ret;
1961
1962         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1963                 return rte_flow_error_set(error, ENOTSUP,
1964                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1965                                           "extended metadata feature"
1966                                           " isn't enabled");
1967         if (!mlx5_flow_ext_mreg_supported(dev))
1968                 return rte_flow_error_set(error, ENOTSUP,
1969                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1970                                           "extended metadata register"
1971                                           " isn't supported");
1972         if (!nic_mask.id)
1973                 return rte_flow_error_set(error, ENOTSUP,
1974                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1975                                           "extended metadata register"
1976                                           " isn't available");
1977         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1978         if (ret < 0)
1979                 return ret;
1980         if (!spec)
1981                 return rte_flow_error_set(error, EINVAL,
1982                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1983                                           item->spec,
1984                                           "data cannot be empty");
1985         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1986                 return rte_flow_error_set(error, EINVAL,
1987                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1988                                           &spec->id,
1989                                           "mark id exceeds the limit");
1990         if (!mask)
1991                 mask = &nic_mask;
1992         if (!mask->id)
1993                 return rte_flow_error_set(error, EINVAL,
1994                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1995                                         "mask cannot be zero");
1996
1997         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1998                                         (const uint8_t *)&nic_mask,
1999                                         sizeof(struct rte_flow_item_mark),
2000                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2001         if (ret < 0)
2002                 return ret;
2003         return 0;
2004 }
2005
2006 /**
2007  * Validate META item.
2008  *
2009  * @param[in] dev
2010  *   Pointer to the rte_eth_dev structure.
2011  * @param[in] item
2012  *   Item specification.
2013  * @param[in] attr
2014  *   Attributes of flow that includes this item.
2015  * @param[out] error
2016  *   Pointer to error structure.
2017  *
2018  * @return
2019  *   0 on success, a negative errno value otherwise and rte_errno is set.
2020  */
2021 static int
2022 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
2023                            const struct rte_flow_item *item,
2024                            const struct rte_flow_attr *attr,
2025                            struct rte_flow_error *error)
2026 {
2027         struct mlx5_priv *priv = dev->data->dev_private;
2028         struct mlx5_dev_config *config = &priv->config;
2029         const struct rte_flow_item_meta *spec = item->spec;
2030         const struct rte_flow_item_meta *mask = item->mask;
2031         struct rte_flow_item_meta nic_mask = {
2032                 .data = UINT32_MAX
2033         };
2034         int reg;
2035         int ret;
2036
2037         if (!spec)
2038                 return rte_flow_error_set(error, EINVAL,
2039                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2040                                           item->spec,
2041                                           "data cannot be empty");
2042         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2043                 if (!mlx5_flow_ext_mreg_supported(dev))
2044                         return rte_flow_error_set(error, ENOTSUP,
2045                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2046                                           "extended metadata register"
2047                                           " isn't supported");
2048                 reg = flow_dv_get_metadata_reg(dev, attr, error);
2049                 if (reg < 0)
2050                         return reg;
2051                 if (reg == REG_NON)
2052                         return rte_flow_error_set(error, ENOTSUP,
2053                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2054                                         "unavalable extended metadata register");
2055                 if (reg == REG_B)
2056                         return rte_flow_error_set(error, ENOTSUP,
2057                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2058                                           "match on reg_b "
2059                                           "isn't supported");
2060                 if (reg != REG_A)
2061                         nic_mask.data = priv->sh->dv_meta_mask;
2062         } else {
2063                 if (attr->transfer)
2064                         return rte_flow_error_set(error, ENOTSUP,
2065                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2066                                         "extended metadata feature "
2067                                         "should be enabled when "
2068                                         "meta item is requested "
2069                                         "with e-switch mode ");
2070                 if (attr->ingress)
2071                         return rte_flow_error_set(error, ENOTSUP,
2072                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2073                                         "match on metadata for ingress "
2074                                         "is not supported in legacy "
2075                                         "metadata mode");
2076         }
2077         if (!mask)
2078                 mask = &rte_flow_item_meta_mask;
2079         if (!mask->data)
2080                 return rte_flow_error_set(error, EINVAL,
2081                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2082                                         "mask cannot be zero");
2083
2084         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2085                                         (const uint8_t *)&nic_mask,
2086                                         sizeof(struct rte_flow_item_meta),
2087                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2088         return ret;
2089 }
2090
2091 /**
2092  * Validate TAG item.
2093  *
2094  * @param[in] dev
2095  *   Pointer to the rte_eth_dev structure.
2096  * @param[in] item
2097  *   Item specification.
2098  * @param[in] attr
2099  *   Attributes of flow that includes this item.
2100  * @param[out] error
2101  *   Pointer to error structure.
2102  *
2103  * @return
2104  *   0 on success, a negative errno value otherwise and rte_errno is set.
2105  */
2106 static int
2107 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
2108                           const struct rte_flow_item *item,
2109                           const struct rte_flow_attr *attr __rte_unused,
2110                           struct rte_flow_error *error)
2111 {
2112         const struct rte_flow_item_tag *spec = item->spec;
2113         const struct rte_flow_item_tag *mask = item->mask;
2114         const struct rte_flow_item_tag nic_mask = {
2115                 .data = RTE_BE32(UINT32_MAX),
2116                 .index = 0xff,
2117         };
2118         int ret;
2119
2120         if (!mlx5_flow_ext_mreg_supported(dev))
2121                 return rte_flow_error_set(error, ENOTSUP,
2122                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2123                                           "extensive metadata register"
2124                                           " isn't supported");
2125         if (!spec)
2126                 return rte_flow_error_set(error, EINVAL,
2127                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2128                                           item->spec,
2129                                           "data cannot be empty");
2130         if (!mask)
2131                 mask = &rte_flow_item_tag_mask;
2132         if (!mask->data)
2133                 return rte_flow_error_set(error, EINVAL,
2134                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2135                                         "mask cannot be zero");
2136
2137         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2138                                         (const uint8_t *)&nic_mask,
2139                                         sizeof(struct rte_flow_item_tag),
2140                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2141         if (ret < 0)
2142                 return ret;
2143         if (mask->index != 0xff)
2144                 return rte_flow_error_set(error, EINVAL,
2145                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2146                                           "partial mask for tag index"
2147                                           " is not supported");
2148         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
2149         if (ret < 0)
2150                 return ret;
2151         MLX5_ASSERT(ret != REG_NON);
2152         return 0;
2153 }
2154
2155 /**
2156  * Validate vport item.
2157  *
2158  * @param[in] dev
2159  *   Pointer to the rte_eth_dev structure.
2160  * @param[in] item
2161  *   Item specification.
2162  * @param[in] attr
2163  *   Attributes of flow that includes this item.
2164  * @param[in] item_flags
2165  *   Bit-fields that holds the items detected until now.
2166  * @param[out] error
2167  *   Pointer to error structure.
2168  *
2169  * @return
2170  *   0 on success, a negative errno value otherwise and rte_errno is set.
2171  */
2172 static int
2173 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
2174                               const struct rte_flow_item *item,
2175                               const struct rte_flow_attr *attr,
2176                               uint64_t item_flags,
2177                               struct rte_flow_error *error)
2178 {
2179         const struct rte_flow_item_port_id *spec = item->spec;
2180         const struct rte_flow_item_port_id *mask = item->mask;
2181         const struct rte_flow_item_port_id switch_mask = {
2182                         .id = 0xffffffff,
2183         };
2184         struct mlx5_priv *esw_priv;
2185         struct mlx5_priv *dev_priv;
2186         int ret;
2187
2188         if (!attr->transfer)
2189                 return rte_flow_error_set(error, EINVAL,
2190                                           RTE_FLOW_ERROR_TYPE_ITEM,
2191                                           NULL,
2192                                           "match on port id is valid only"
2193                                           " when transfer flag is enabled");
2194         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
2195                 return rte_flow_error_set(error, ENOTSUP,
2196                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2197                                           "multiple source ports are not"
2198                                           " supported");
2199         if (!mask)
2200                 mask = &switch_mask;
2201         if (mask->id != 0xffffffff)
2202                 return rte_flow_error_set(error, ENOTSUP,
2203                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2204                                            mask,
2205                                            "no support for partial mask on"
2206                                            " \"id\" field");
2207         ret = mlx5_flow_item_acceptable
2208                                 (item, (const uint8_t *)mask,
2209                                  (const uint8_t *)&rte_flow_item_port_id_mask,
2210                                  sizeof(struct rte_flow_item_port_id),
2211                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2212         if (ret)
2213                 return ret;
2214         if (!spec)
2215                 return 0;
2216         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
2217         if (!esw_priv)
2218                 return rte_flow_error_set(error, rte_errno,
2219                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2220                                           "failed to obtain E-Switch info for"
2221                                           " port");
2222         dev_priv = mlx5_dev_to_eswitch_info(dev);
2223         if (!dev_priv)
2224                 return rte_flow_error_set(error, rte_errno,
2225                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2226                                           NULL,
2227                                           "failed to obtain E-Switch info");
2228         if (esw_priv->domain_id != dev_priv->domain_id)
2229                 return rte_flow_error_set(error, EINVAL,
2230                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2231                                           "cannot match on a port from a"
2232                                           " different E-Switch");
2233         return 0;
2234 }
2235
2236 /**
2237  * Validate VLAN item.
2238  *
2239  * @param[in] item
2240  *   Item specification.
2241  * @param[in] item_flags
2242  *   Bit-fields that holds the items detected until now.
2243  * @param[in] dev
2244  *   Ethernet device flow is being created on.
2245  * @param[out] error
2246  *   Pointer to error structure.
2247  *
2248  * @return
2249  *   0 on success, a negative errno value otherwise and rte_errno is set.
2250  */
2251 static int
2252 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
2253                            uint64_t item_flags,
2254                            struct rte_eth_dev *dev,
2255                            struct rte_flow_error *error)
2256 {
2257         const struct rte_flow_item_vlan *mask = item->mask;
2258         const struct rte_flow_item_vlan nic_mask = {
2259                 .tci = RTE_BE16(UINT16_MAX),
2260                 .inner_type = RTE_BE16(UINT16_MAX),
2261                 .has_more_vlan = 1,
2262         };
2263         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2264         int ret;
2265         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2266                                         MLX5_FLOW_LAYER_INNER_L4) :
2267                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2268                                         MLX5_FLOW_LAYER_OUTER_L4);
2269         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2270                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2271
2272         if (item_flags & vlanm)
2273                 return rte_flow_error_set(error, EINVAL,
2274                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2275                                           "multiple VLAN layers not supported");
2276         else if ((item_flags & l34m) != 0)
2277                 return rte_flow_error_set(error, EINVAL,
2278                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2279                                           "VLAN cannot follow L3/L4 layer");
2280         if (!mask)
2281                 mask = &rte_flow_item_vlan_mask;
2282         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2283                                         (const uint8_t *)&nic_mask,
2284                                         sizeof(struct rte_flow_item_vlan),
2285                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2286         if (ret)
2287                 return ret;
2288         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2289                 struct mlx5_priv *priv = dev->data->dev_private;
2290
2291                 if (priv->vmwa_context) {
2292                         /*
2293                          * Non-NULL context means we have a virtual machine
2294                          * and SR-IOV enabled, we have to create VLAN interface
2295                          * to make hypervisor to setup E-Switch vport
2296                          * context correctly. We avoid creating the multiple
2297                          * VLAN interfaces, so we cannot support VLAN tag mask.
2298                          */
2299                         return rte_flow_error_set(error, EINVAL,
2300                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2301                                                   item,
2302                                                   "VLAN tag mask is not"
2303                                                   " supported in virtual"
2304                                                   " environment");
2305                 }
2306         }
2307         return 0;
2308 }
2309
2310 /*
2311  * GTP flags are contained in 1 byte of the format:
2312  * -------------------------------------------
2313  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
2314  * |-----------------------------------------|
2315  * | value | Version | PT | Res | E | S | PN |
2316  * -------------------------------------------
2317  *
2318  * Matching is supported only for GTP flags E, S, PN.
2319  */
2320 #define MLX5_GTP_FLAGS_MASK     0x07
2321
2322 /**
2323  * Validate GTP item.
2324  *
2325  * @param[in] dev
2326  *   Pointer to the rte_eth_dev structure.
2327  * @param[in] item
2328  *   Item specification.
2329  * @param[in] item_flags
2330  *   Bit-fields that holds the items detected until now.
2331  * @param[out] error
2332  *   Pointer to error structure.
2333  *
2334  * @return
2335  *   0 on success, a negative errno value otherwise and rte_errno is set.
2336  */
2337 static int
2338 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
2339                           const struct rte_flow_item *item,
2340                           uint64_t item_flags,
2341                           struct rte_flow_error *error)
2342 {
2343         struct mlx5_priv *priv = dev->data->dev_private;
2344         const struct rte_flow_item_gtp *spec = item->spec;
2345         const struct rte_flow_item_gtp *mask = item->mask;
2346         const struct rte_flow_item_gtp nic_mask = {
2347                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
2348                 .msg_type = 0xff,
2349                 .teid = RTE_BE32(0xffffffff),
2350         };
2351
2352         if (!priv->config.hca_attr.tunnel_stateless_gtp)
2353                 return rte_flow_error_set(error, ENOTSUP,
2354                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2355                                           "GTP support is not enabled");
2356         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2357                 return rte_flow_error_set(error, ENOTSUP,
2358                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2359                                           "multiple tunnel layers not"
2360                                           " supported");
2361         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2362                 return rte_flow_error_set(error, EINVAL,
2363                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2364                                           "no outer UDP layer found");
2365         if (!mask)
2366                 mask = &rte_flow_item_gtp_mask;
2367         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
2368                 return rte_flow_error_set(error, ENOTSUP,
2369                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2370                                           "Match is supported for GTP"
2371                                           " flags only");
2372         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2373                                          (const uint8_t *)&nic_mask,
2374                                          sizeof(struct rte_flow_item_gtp),
2375                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2376 }
2377
2378 /**
2379  * Validate GTP PSC item.
2380  *
2381  * @param[in] item
2382  *   Item specification.
2383  * @param[in] last_item
2384  *   Previous validated item in the pattern items.
2385  * @param[in] gtp_item
2386  *   Previous GTP item specification.
2387  * @param[in] attr
2388  *   Pointer to flow attributes.
2389  * @param[out] error
2390  *   Pointer to error structure.
2391  *
2392  * @return
2393  *   0 on success, a negative errno value otherwise and rte_errno is set.
2394  */
2395 static int
2396 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
2397                               uint64_t last_item,
2398                               const struct rte_flow_item *gtp_item,
2399                               const struct rte_flow_attr *attr,
2400                               struct rte_flow_error *error)
2401 {
2402         const struct rte_flow_item_gtp *gtp_spec;
2403         const struct rte_flow_item_gtp *gtp_mask;
2404         const struct rte_flow_item_gtp_psc *spec;
2405         const struct rte_flow_item_gtp_psc *mask;
2406         const struct rte_flow_item_gtp_psc nic_mask = {
2407                 .pdu_type = 0xFF,
2408                 .qfi = 0xFF,
2409         };
2410
2411         if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
2412                 return rte_flow_error_set
2413                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2414                          "GTP PSC item must be preceded with GTP item");
2415         gtp_spec = gtp_item->spec;
2416         gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
2417         /* GTP spec and E flag is requested to match zero. */
2418         if (gtp_spec &&
2419                 (gtp_mask->v_pt_rsv_flags &
2420                 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
2421                 return rte_flow_error_set
2422                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2423                          "GTP E flag must be 1 to match GTP PSC");
2424         /* Check the flow is not created in group zero. */
2425         if (!attr->transfer && !attr->group)
2426                 return rte_flow_error_set
2427                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2428                          "GTP PSC is not supported for group 0");
2429         /* GTP spec is here and E flag is requested to match zero. */
2430         if (!item->spec)
2431                 return 0;
2432         spec = item->spec;
2433         mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
2434         if (spec->pdu_type > MLX5_GTP_EXT_MAX_PDU_TYPE)
2435                 return rte_flow_error_set
2436                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2437                          "PDU type should be smaller than 16");
2438         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2439                                          (const uint8_t *)&nic_mask,
2440                                          sizeof(struct rte_flow_item_gtp_psc),
2441                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2442 }
2443
2444 /**
2445  * Validate IPV4 item.
2446  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
2447  * add specific validation of fragment_offset field,
2448  *
2449  * @param[in] item
2450  *   Item specification.
2451  * @param[in] item_flags
2452  *   Bit-fields that holds the items detected until now.
2453  * @param[out] error
2454  *   Pointer to error structure.
2455  *
2456  * @return
2457  *   0 on success, a negative errno value otherwise and rte_errno is set.
2458  */
2459 static int
2460 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
2461                            uint64_t item_flags,
2462                            uint64_t last_item,
2463                            uint16_t ether_type,
2464                            struct rte_flow_error *error)
2465 {
2466         int ret;
2467         const struct rte_flow_item_ipv4 *spec = item->spec;
2468         const struct rte_flow_item_ipv4 *last = item->last;
2469         const struct rte_flow_item_ipv4 *mask = item->mask;
2470         rte_be16_t fragment_offset_spec = 0;
2471         rte_be16_t fragment_offset_last = 0;
2472         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
2473                 .hdr = {
2474                         .src_addr = RTE_BE32(0xffffffff),
2475                         .dst_addr = RTE_BE32(0xffffffff),
2476                         .type_of_service = 0xff,
2477                         .fragment_offset = RTE_BE16(0xffff),
2478                         .next_proto_id = 0xff,
2479                         .time_to_live = 0xff,
2480                 },
2481         };
2482
2483         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
2484                                            ether_type, &nic_ipv4_mask,
2485                                            MLX5_ITEM_RANGE_ACCEPTED, error);
2486         if (ret < 0)
2487                 return ret;
2488         if (spec && mask)
2489                 fragment_offset_spec = spec->hdr.fragment_offset &
2490                                        mask->hdr.fragment_offset;
2491         if (!fragment_offset_spec)
2492                 return 0;
2493         /*
2494          * spec and mask are valid, enforce using full mask to make sure the
2495          * complete value is used correctly.
2496          */
2497         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2498                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2499                 return rte_flow_error_set(error, EINVAL,
2500                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2501                                           item, "must use full mask for"
2502                                           " fragment_offset");
2503         /*
2504          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
2505          * indicating this is 1st fragment of fragmented packet.
2506          * This is not yet supported in MLX5, return appropriate error message.
2507          */
2508         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
2509                 return rte_flow_error_set(error, ENOTSUP,
2510                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2511                                           "match on first fragment not "
2512                                           "supported");
2513         if (fragment_offset_spec && !last)
2514                 return rte_flow_error_set(error, ENOTSUP,
2515                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2516                                           "specified value not supported");
2517         /* spec and last are valid, validate the specified range. */
2518         fragment_offset_last = last->hdr.fragment_offset &
2519                                mask->hdr.fragment_offset;
2520         /*
2521          * Match on fragment_offset spec 0x2001 and last 0x3fff
2522          * means MF is 1 and frag-offset is > 0.
2523          * This packet is fragment 2nd and onward, excluding last.
2524          * This is not yet supported in MLX5, return appropriate
2525          * error message.
2526          */
2527         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
2528             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2529                 return rte_flow_error_set(error, ENOTSUP,
2530                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2531                                           last, "match on following "
2532                                           "fragments not supported");
2533         /*
2534          * Match on fragment_offset spec 0x0001 and last 0x1fff
2535          * means MF is 0 and frag-offset is > 0.
2536          * This packet is last fragment of fragmented packet.
2537          * This is not yet supported in MLX5, return appropriate
2538          * error message.
2539          */
2540         if (fragment_offset_spec == RTE_BE16(1) &&
2541             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
2542                 return rte_flow_error_set(error, ENOTSUP,
2543                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2544                                           last, "match on last "
2545                                           "fragment not supported");
2546         /*
2547          * Match on fragment_offset spec 0x0001 and last 0x3fff
2548          * means MF and/or frag-offset is not 0.
2549          * This is a fragmented packet.
2550          * Other range values are invalid and rejected.
2551          */
2552         if (!(fragment_offset_spec == RTE_BE16(1) &&
2553               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
2554                 return rte_flow_error_set(error, ENOTSUP,
2555                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2556                                           "specified range not supported");
2557         return 0;
2558 }
2559
2560 /**
2561  * Validate IPV6 fragment extension item.
2562  *
2563  * @param[in] item
2564  *   Item specification.
2565  * @param[in] item_flags
2566  *   Bit-fields that holds the items detected until now.
2567  * @param[out] error
2568  *   Pointer to error structure.
2569  *
2570  * @return
2571  *   0 on success, a negative errno value otherwise and rte_errno is set.
2572  */
2573 static int
2574 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
2575                                     uint64_t item_flags,
2576                                     struct rte_flow_error *error)
2577 {
2578         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
2579         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
2580         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
2581         rte_be16_t frag_data_spec = 0;
2582         rte_be16_t frag_data_last = 0;
2583         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2584         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2585                                       MLX5_FLOW_LAYER_OUTER_L4;
2586         int ret = 0;
2587         struct rte_flow_item_ipv6_frag_ext nic_mask = {
2588                 .hdr = {
2589                         .next_header = 0xff,
2590                         .frag_data = RTE_BE16(0xffff),
2591                 },
2592         };
2593
2594         if (item_flags & l4m)
2595                 return rte_flow_error_set(error, EINVAL,
2596                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2597                                           "ipv6 fragment extension item cannot "
2598                                           "follow L4 item.");
2599         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
2600             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2601                 return rte_flow_error_set(error, EINVAL,
2602                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2603                                           "ipv6 fragment extension item must "
2604                                           "follow ipv6 item");
2605         if (spec && mask)
2606                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
2607         if (!frag_data_spec)
2608                 return 0;
2609         /*
2610          * spec and mask are valid, enforce using full mask to make sure the
2611          * complete value is used correctly.
2612          */
2613         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
2614                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2615                 return rte_flow_error_set(error, EINVAL,
2616                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2617                                           item, "must use full mask for"
2618                                           " frag_data");
2619         /*
2620          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
2621          * This is 1st fragment of fragmented packet.
2622          */
2623         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
2624                 return rte_flow_error_set(error, ENOTSUP,
2625                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2626                                           "match on first fragment not "
2627                                           "supported");
2628         if (frag_data_spec && !last)
2629                 return rte_flow_error_set(error, EINVAL,
2630                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2631                                           "specified value not supported");
2632         ret = mlx5_flow_item_acceptable
2633                                 (item, (const uint8_t *)mask,
2634                                  (const uint8_t *)&nic_mask,
2635                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
2636                                  MLX5_ITEM_RANGE_ACCEPTED, error);
2637         if (ret)
2638                 return ret;
2639         /* spec and last are valid, validate the specified range. */
2640         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
2641         /*
2642          * Match on frag_data spec 0x0009 and last 0xfff9
2643          * means M is 1 and frag-offset is > 0.
2644          * This packet is fragment 2nd and onward, excluding last.
2645          * This is not yet supported in MLX5, return appropriate
2646          * error message.
2647          */
2648         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2649                                        RTE_IPV6_EHDR_MF_MASK) &&
2650             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2651                 return rte_flow_error_set(error, ENOTSUP,
2652                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2653                                           last, "match on following "
2654                                           "fragments not supported");
2655         /*
2656          * Match on frag_data spec 0x0008 and last 0xfff8
2657          * means M is 0 and frag-offset is > 0.
2658          * This packet is last fragment of fragmented packet.
2659          * This is not yet supported in MLX5, return appropriate
2660          * error message.
2661          */
2662         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2663             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2664                 return rte_flow_error_set(error, ENOTSUP,
2665                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2666                                           last, "match on last "
2667                                           "fragment not supported");
2668         /* Other range values are invalid and rejected. */
2669         return rte_flow_error_set(error, EINVAL,
2670                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2671                                   "specified range not supported");
2672 }
2673
2674 /*
2675  * Validate ASO CT item.
2676  *
2677  * @param[in] dev
2678  *   Pointer to the rte_eth_dev structure.
2679  * @param[in] item
2680  *   Item specification.
2681  * @param[in] item_flags
2682  *   Pointer to bit-fields that holds the items detected until now.
2683  * @param[out] error
2684  *   Pointer to error structure.
2685  *
2686  * @return
2687  *   0 on success, a negative errno value otherwise and rte_errno is set.
2688  */
2689 static int
2690 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
2691                              const struct rte_flow_item *item,
2692                              uint64_t *item_flags,
2693                              struct rte_flow_error *error)
2694 {
2695         const struct rte_flow_item_conntrack *spec = item->spec;
2696         const struct rte_flow_item_conntrack *mask = item->mask;
2697         RTE_SET_USED(dev);
2698         uint32_t flags;
2699
2700         if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
2701                 return rte_flow_error_set(error, EINVAL,
2702                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2703                                           "Only one CT is supported");
2704         if (!mask)
2705                 mask = &rte_flow_item_conntrack_mask;
2706         flags = spec->flags & mask->flags;
2707         if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
2708             ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
2709              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
2710              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
2711                 return rte_flow_error_set(error, EINVAL,
2712                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2713                                           "Conflict status bits");
2714         /* State change also needs to be considered. */
2715         *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
2716         return 0;
2717 }
2718
2719 /**
2720  * Validate the pop VLAN action.
2721  *
2722  * @param[in] dev
2723  *   Pointer to the rte_eth_dev structure.
2724  * @param[in] action_flags
2725  *   Holds the actions detected until now.
2726  * @param[in] action
2727  *   Pointer to the pop vlan action.
2728  * @param[in] item_flags
2729  *   The items found in this flow rule.
2730  * @param[in] attr
2731  *   Pointer to flow attributes.
2732  * @param[out] error
2733  *   Pointer to error structure.
2734  *
2735  * @return
2736  *   0 on success, a negative errno value otherwise and rte_errno is set.
2737  */
2738 static int
2739 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2740                                  uint64_t action_flags,
2741                                  const struct rte_flow_action *action,
2742                                  uint64_t item_flags,
2743                                  const struct rte_flow_attr *attr,
2744                                  struct rte_flow_error *error)
2745 {
2746         const struct mlx5_priv *priv = dev->data->dev_private;
2747
2748         (void)action;
2749         (void)attr;
2750         if (!priv->sh->pop_vlan_action)
2751                 return rte_flow_error_set(error, ENOTSUP,
2752                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2753                                           NULL,
2754                                           "pop vlan action is not supported");
2755         if (attr->egress)
2756                 return rte_flow_error_set(error, ENOTSUP,
2757                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2758                                           NULL,
2759                                           "pop vlan action not supported for "
2760                                           "egress");
2761         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2762                 return rte_flow_error_set(error, ENOTSUP,
2763                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2764                                           "no support for multiple VLAN "
2765                                           "actions");
2766         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2767         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2768             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2769                 return rte_flow_error_set(error, ENOTSUP,
2770                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2771                                           NULL,
2772                                           "cannot pop vlan after decap without "
2773                                           "match on inner vlan in the flow");
2774         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2775         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2776             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2777                 return rte_flow_error_set(error, ENOTSUP,
2778                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2779                                           NULL,
2780                                           "cannot pop vlan without a "
2781                                           "match on (outer) vlan in the flow");
2782         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2783                 return rte_flow_error_set(error, EINVAL,
2784                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2785                                           "wrong action order, port_id should "
2786                                           "be after pop VLAN action");
2787         if (!attr->transfer && priv->representor)
2788                 return rte_flow_error_set(error, ENOTSUP,
2789                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2790                                           "pop vlan action for VF representor "
2791                                           "not supported on NIC table");
2792         return 0;
2793 }
2794
2795 /**
2796  * Get VLAN default info from vlan match info.
2797  *
2798  * @param[in] items
2799  *   the list of item specifications.
2800  * @param[out] vlan
2801  *   pointer VLAN info to fill to.
2802  *
2803  * @return
2804  *   0 on success, a negative errno value otherwise and rte_errno is set.
2805  */
2806 static void
2807 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2808                                   struct rte_vlan_hdr *vlan)
2809 {
2810         const struct rte_flow_item_vlan nic_mask = {
2811                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2812                                 MLX5DV_FLOW_VLAN_VID_MASK),
2813                 .inner_type = RTE_BE16(0xffff),
2814         };
2815
2816         if (items == NULL)
2817                 return;
2818         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2819                 int type = items->type;
2820
2821                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2822                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2823                         break;
2824         }
2825         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2826                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2827                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2828
2829                 /* If VLAN item in pattern doesn't contain data, return here. */
2830                 if (!vlan_v)
2831                         return;
2832                 if (!vlan_m)
2833                         vlan_m = &nic_mask;
2834                 /* Only full match values are accepted */
2835                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2836                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2837                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2838                         vlan->vlan_tci |=
2839                                 rte_be_to_cpu_16(vlan_v->tci &
2840                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2841                 }
2842                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2843                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2844                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2845                         vlan->vlan_tci |=
2846                                 rte_be_to_cpu_16(vlan_v->tci &
2847                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2848                 }
2849                 if (vlan_m->inner_type == nic_mask.inner_type)
2850                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2851                                                            vlan_m->inner_type);
2852         }
2853 }
2854
2855 /**
2856  * Validate the push VLAN action.
2857  *
2858  * @param[in] dev
2859  *   Pointer to the rte_eth_dev structure.
2860  * @param[in] action_flags
2861  *   Holds the actions detected until now.
2862  * @param[in] item_flags
2863  *   The items found in this flow rule.
2864  * @param[in] action
2865  *   Pointer to the action structure.
2866  * @param[in] attr
2867  *   Pointer to flow attributes
2868  * @param[out] error
2869  *   Pointer to error structure.
2870  *
2871  * @return
2872  *   0 on success, a negative errno value otherwise and rte_errno is set.
2873  */
2874 static int
2875 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2876                                   uint64_t action_flags,
2877                                   const struct rte_flow_item_vlan *vlan_m,
2878                                   const struct rte_flow_action *action,
2879                                   const struct rte_flow_attr *attr,
2880                                   struct rte_flow_error *error)
2881 {
2882         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2883         const struct mlx5_priv *priv = dev->data->dev_private;
2884
2885         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2886             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2887                 return rte_flow_error_set(error, EINVAL,
2888                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2889                                           "invalid vlan ethertype");
2890         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2891                 return rte_flow_error_set(error, EINVAL,
2892                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2893                                           "wrong action order, port_id should "
2894                                           "be after push VLAN");
2895         if (!attr->transfer && priv->representor)
2896                 return rte_flow_error_set(error, ENOTSUP,
2897                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2898                                           "push vlan action for VF representor "
2899                                           "not supported on NIC table");
2900         if (vlan_m &&
2901             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2902             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2903                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2904             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2905             !(mlx5_flow_find_action
2906                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2907                 return rte_flow_error_set(error, EINVAL,
2908                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2909                                           "not full match mask on VLAN PCP and "
2910                                           "there is no of_set_vlan_pcp action, "
2911                                           "push VLAN action cannot figure out "
2912                                           "PCP value");
2913         if (vlan_m &&
2914             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2915             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2916                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2917             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2918             !(mlx5_flow_find_action
2919                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2920                 return rte_flow_error_set(error, EINVAL,
2921                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2922                                           "not full match mask on VLAN VID and "
2923                                           "there is no of_set_vlan_vid action, "
2924                                           "push VLAN action cannot figure out "
2925                                           "VID value");
2926         (void)attr;
2927         return 0;
2928 }
2929
2930 /**
2931  * Validate the set VLAN PCP.
2932  *
2933  * @param[in] action_flags
2934  *   Holds the actions detected until now.
2935  * @param[in] actions
2936  *   Pointer to the list of actions remaining in the flow rule.
2937  * @param[out] error
2938  *   Pointer to error structure.
2939  *
2940  * @return
2941  *   0 on success, a negative errno value otherwise and rte_errno is set.
2942  */
2943 static int
2944 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2945                                      const struct rte_flow_action actions[],
2946                                      struct rte_flow_error *error)
2947 {
2948         const struct rte_flow_action *action = actions;
2949         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2950
2951         if (conf->vlan_pcp > 7)
2952                 return rte_flow_error_set(error, EINVAL,
2953                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2954                                           "VLAN PCP value is too big");
2955         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2956                 return rte_flow_error_set(error, ENOTSUP,
2957                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2958                                           "set VLAN PCP action must follow "
2959                                           "the push VLAN action");
2960         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2961                 return rte_flow_error_set(error, ENOTSUP,
2962                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2963                                           "Multiple VLAN PCP modification are "
2964                                           "not supported");
2965         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2966                 return rte_flow_error_set(error, EINVAL,
2967                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2968                                           "wrong action order, port_id should "
2969                                           "be after set VLAN PCP");
2970         return 0;
2971 }
2972
2973 /**
2974  * Validate the set VLAN VID.
2975  *
2976  * @param[in] item_flags
2977  *   Holds the items detected in this rule.
2978  * @param[in] action_flags
2979  *   Holds the actions detected until now.
2980  * @param[in] actions
2981  *   Pointer to the list of actions remaining in the flow rule.
2982  * @param[out] error
2983  *   Pointer to error structure.
2984  *
2985  * @return
2986  *   0 on success, a negative errno value otherwise and rte_errno is set.
2987  */
2988 static int
2989 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2990                                      uint64_t action_flags,
2991                                      const struct rte_flow_action actions[],
2992                                      struct rte_flow_error *error)
2993 {
2994         const struct rte_flow_action *action = actions;
2995         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2996
2997         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2998                 return rte_flow_error_set(error, EINVAL,
2999                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3000                                           "VLAN VID value is too big");
3001         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
3002             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
3003                 return rte_flow_error_set(error, ENOTSUP,
3004                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3005                                           "set VLAN VID action must follow push"
3006                                           " VLAN action or match on VLAN item");
3007         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
3008                 return rte_flow_error_set(error, ENOTSUP,
3009                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3010                                           "Multiple VLAN VID modifications are "
3011                                           "not supported");
3012         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3013                 return rte_flow_error_set(error, EINVAL,
3014                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3015                                           "wrong action order, port_id should "
3016                                           "be after set VLAN VID");
3017         return 0;
3018 }
3019
3020 /*
3021  * Validate the FLAG action.
3022  *
3023  * @param[in] dev
3024  *   Pointer to the rte_eth_dev structure.
3025  * @param[in] action_flags
3026  *   Holds the actions detected until now.
3027  * @param[in] attr
3028  *   Pointer to flow attributes
3029  * @param[out] error
3030  *   Pointer to error structure.
3031  *
3032  * @return
3033  *   0 on success, a negative errno value otherwise and rte_errno is set.
3034  */
3035 static int
3036 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
3037                              uint64_t action_flags,
3038                              const struct rte_flow_attr *attr,
3039                              struct rte_flow_error *error)
3040 {
3041         struct mlx5_priv *priv = dev->data->dev_private;
3042         struct mlx5_dev_config *config = &priv->config;
3043         int ret;
3044
3045         /* Fall back if no extended metadata register support. */
3046         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3047                 return mlx5_flow_validate_action_flag(action_flags, attr,
3048                                                       error);
3049         /* Extensive metadata mode requires registers. */
3050         if (!mlx5_flow_ext_mreg_supported(dev))
3051                 return rte_flow_error_set(error, ENOTSUP,
3052                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3053                                           "no metadata registers "
3054                                           "to support flag action");
3055         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
3056                 return rte_flow_error_set(error, ENOTSUP,
3057                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3058                                           "extended metadata register"
3059                                           " isn't available");
3060         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3061         if (ret < 0)
3062                 return ret;
3063         MLX5_ASSERT(ret > 0);
3064         if (action_flags & MLX5_FLOW_ACTION_MARK)
3065                 return rte_flow_error_set(error, EINVAL,
3066                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3067                                           "can't mark and flag in same flow");
3068         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3069                 return rte_flow_error_set(error, EINVAL,
3070                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3071                                           "can't have 2 flag"
3072                                           " actions in same flow");
3073         return 0;
3074 }
3075
3076 /**
3077  * Validate MARK action.
3078  *
3079  * @param[in] dev
3080  *   Pointer to the rte_eth_dev structure.
3081  * @param[in] action
3082  *   Pointer to action.
3083  * @param[in] action_flags
3084  *   Holds the actions detected until now.
3085  * @param[in] attr
3086  *   Pointer to flow attributes
3087  * @param[out] error
3088  *   Pointer to error structure.
3089  *
3090  * @return
3091  *   0 on success, a negative errno value otherwise and rte_errno is set.
3092  */
3093 static int
3094 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
3095                              const struct rte_flow_action *action,
3096                              uint64_t action_flags,
3097                              const struct rte_flow_attr *attr,
3098                              struct rte_flow_error *error)
3099 {
3100         struct mlx5_priv *priv = dev->data->dev_private;
3101         struct mlx5_dev_config *config = &priv->config;
3102         const struct rte_flow_action_mark *mark = action->conf;
3103         int ret;
3104
3105         if (is_tunnel_offload_active(dev))
3106                 return rte_flow_error_set(error, ENOTSUP,
3107                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3108                                           "no mark action "
3109                                           "if tunnel offload active");
3110         /* Fall back if no extended metadata register support. */
3111         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3112                 return mlx5_flow_validate_action_mark(action, action_flags,
3113                                                       attr, error);
3114         /* Extensive metadata mode requires registers. */
3115         if (!mlx5_flow_ext_mreg_supported(dev))
3116                 return rte_flow_error_set(error, ENOTSUP,
3117                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3118                                           "no metadata registers "
3119                                           "to support mark action");
3120         if (!priv->sh->dv_mark_mask)
3121                 return rte_flow_error_set(error, ENOTSUP,
3122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3123                                           "extended metadata register"
3124                                           " isn't available");
3125         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3126         if (ret < 0)
3127                 return ret;
3128         MLX5_ASSERT(ret > 0);
3129         if (!mark)
3130                 return rte_flow_error_set(error, EINVAL,
3131                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3132                                           "configuration cannot be null");
3133         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
3134                 return rte_flow_error_set(error, EINVAL,
3135                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3136                                           &mark->id,
3137                                           "mark id exceeds the limit");
3138         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3139                 return rte_flow_error_set(error, EINVAL,
3140                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3141                                           "can't flag and mark in same flow");
3142         if (action_flags & MLX5_FLOW_ACTION_MARK)
3143                 return rte_flow_error_set(error, EINVAL,
3144                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3145                                           "can't have 2 mark actions in same"
3146                                           " flow");
3147         return 0;
3148 }
3149
3150 /**
3151  * Validate SET_META action.
3152  *
3153  * @param[in] dev
3154  *   Pointer to the rte_eth_dev structure.
3155  * @param[in] action
3156  *   Pointer to the action structure.
3157  * @param[in] action_flags
3158  *   Holds the actions detected until now.
3159  * @param[in] attr
3160  *   Pointer to flow attributes
3161  * @param[out] error
3162  *   Pointer to error structure.
3163  *
3164  * @return
3165  *   0 on success, a negative errno value otherwise and rte_errno is set.
3166  */
3167 static int
3168 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
3169                                  const struct rte_flow_action *action,
3170                                  uint64_t action_flags __rte_unused,
3171                                  const struct rte_flow_attr *attr,
3172                                  struct rte_flow_error *error)
3173 {
3174         const struct rte_flow_action_set_meta *conf;
3175         uint32_t nic_mask = UINT32_MAX;
3176         int reg;
3177
3178         if (!mlx5_flow_ext_mreg_supported(dev))
3179                 return rte_flow_error_set(error, ENOTSUP,
3180                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3181                                           "extended metadata register"
3182                                           " isn't supported");
3183         reg = flow_dv_get_metadata_reg(dev, attr, error);
3184         if (reg < 0)
3185                 return reg;
3186         if (reg == REG_NON)
3187                 return rte_flow_error_set(error, ENOTSUP,
3188                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3189                                           "unavalable extended metadata register");
3190         if (reg != REG_A && reg != REG_B) {
3191                 struct mlx5_priv *priv = dev->data->dev_private;
3192
3193                 nic_mask = priv->sh->dv_meta_mask;
3194         }
3195         if (!(action->conf))
3196                 return rte_flow_error_set(error, EINVAL,
3197                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3198                                           "configuration cannot be null");
3199         conf = (const struct rte_flow_action_set_meta *)action->conf;
3200         if (!conf->mask)
3201                 return rte_flow_error_set(error, EINVAL,
3202                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3203                                           "zero mask doesn't have any effect");
3204         if (conf->mask & ~nic_mask)
3205                 return rte_flow_error_set(error, EINVAL,
3206                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3207                                           "meta data must be within reg C0");
3208         return 0;
3209 }
3210
3211 /**
3212  * Validate SET_TAG action.
3213  *
3214  * @param[in] dev
3215  *   Pointer to the rte_eth_dev structure.
3216  * @param[in] action
3217  *   Pointer to the action structure.
3218  * @param[in] action_flags
3219  *   Holds the actions detected until now.
3220  * @param[in] attr
3221  *   Pointer to flow attributes
3222  * @param[out] error
3223  *   Pointer to error structure.
3224  *
3225  * @return
3226  *   0 on success, a negative errno value otherwise and rte_errno is set.
3227  */
3228 static int
3229 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
3230                                 const struct rte_flow_action *action,
3231                                 uint64_t action_flags,
3232                                 const struct rte_flow_attr *attr,
3233                                 struct rte_flow_error *error)
3234 {
3235         const struct rte_flow_action_set_tag *conf;
3236         const uint64_t terminal_action_flags =
3237                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
3238                 MLX5_FLOW_ACTION_RSS;
3239         int ret;
3240
3241         if (!mlx5_flow_ext_mreg_supported(dev))
3242                 return rte_flow_error_set(error, ENOTSUP,
3243                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3244                                           "extensive metadata register"
3245                                           " isn't supported");
3246         if (!(action->conf))
3247                 return rte_flow_error_set(error, EINVAL,
3248                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3249                                           "configuration cannot be null");
3250         conf = (const struct rte_flow_action_set_tag *)action->conf;
3251         if (!conf->mask)
3252                 return rte_flow_error_set(error, EINVAL,
3253                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3254                                           "zero mask doesn't have any effect");
3255         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
3256         if (ret < 0)
3257                 return ret;
3258         if (!attr->transfer && attr->ingress &&
3259             (action_flags & terminal_action_flags))
3260                 return rte_flow_error_set(error, EINVAL,
3261                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3262                                           "set_tag has no effect"
3263                                           " with terminal actions");
3264         return 0;
3265 }
3266
3267 /**
3268  * Check if action counter is shared by either old or new mechanism.
3269  *
3270  * @param[in] action
3271  *   Pointer to the action structure.
3272  *
3273  * @return
3274  *   True when counter is shared, false otherwise.
3275  */
3276 static inline bool
3277 is_shared_action_count(const struct rte_flow_action *action)
3278 {
3279         const struct rte_flow_action_count *count =
3280                         (const struct rte_flow_action_count *)action->conf;
3281
3282         if ((int)action->type == MLX5_RTE_FLOW_ACTION_TYPE_COUNT)
3283                 return true;
3284         return !!(count && count->shared);
3285 }
3286
3287 /**
3288  * Validate count action.
3289  *
3290  * @param[in] dev
3291  *   Pointer to rte_eth_dev structure.
3292  * @param[in] shared
3293  *   Indicator if action is shared.
3294  * @param[in] action_flags
3295  *   Holds the actions detected until now.
3296  * @param[out] error
3297  *   Pointer to error structure.
3298  *
3299  * @return
3300  *   0 on success, a negative errno value otherwise and rte_errno is set.
3301  */
3302 static int
3303 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
3304                               uint64_t action_flags,
3305                               struct rte_flow_error *error)
3306 {
3307         struct mlx5_priv *priv = dev->data->dev_private;
3308
3309         if (!priv->config.devx)
3310                 goto notsup_err;
3311         if (action_flags & MLX5_FLOW_ACTION_COUNT)
3312                 return rte_flow_error_set(error, EINVAL,
3313                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3314                                           "duplicate count actions set");
3315         if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
3316             !priv->sh->flow_hit_aso_en)
3317                 return rte_flow_error_set(error, EINVAL,
3318                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3319                                           "old age and shared count combination is not supported");
3320 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
3321         return 0;
3322 #endif
3323 notsup_err:
3324         return rte_flow_error_set
3325                       (error, ENOTSUP,
3326                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3327                        NULL,
3328                        "count action not supported");
3329 }
3330
3331 /**
3332  * Validate the L2 encap action.
3333  *
3334  * @param[in] dev
3335  *   Pointer to the rte_eth_dev structure.
3336  * @param[in] action_flags
3337  *   Holds the actions detected until now.
3338  * @param[in] action
3339  *   Pointer to the action structure.
3340  * @param[in] attr
3341  *   Pointer to flow attributes.
3342  * @param[out] error
3343  *   Pointer to error structure.
3344  *
3345  * @return
3346  *   0 on success, a negative errno value otherwise and rte_errno is set.
3347  */
3348 static int
3349 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
3350                                  uint64_t action_flags,
3351                                  const struct rte_flow_action *action,
3352                                  const struct rte_flow_attr *attr,
3353                                  struct rte_flow_error *error)
3354 {
3355         const struct mlx5_priv *priv = dev->data->dev_private;
3356
3357         if (!(action->conf))
3358                 return rte_flow_error_set(error, EINVAL,
3359                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3360                                           "configuration cannot be null");
3361         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3362                 return rte_flow_error_set(error, EINVAL,
3363                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3364                                           "can only have a single encap action "
3365                                           "in a flow");
3366         if (!attr->transfer && priv->representor)
3367                 return rte_flow_error_set(error, ENOTSUP,
3368                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3369                                           "encap action for VF representor "
3370                                           "not supported on NIC table");
3371         return 0;
3372 }
3373
3374 /**
3375  * Validate a decap action.
3376  *
3377  * @param[in] dev
3378  *   Pointer to the rte_eth_dev structure.
3379  * @param[in] action_flags
3380  *   Holds the actions detected until now.
3381  * @param[in] action
3382  *   Pointer to the action structure.
3383  * @param[in] item_flags
3384  *   Holds the items detected.
3385  * @param[in] attr
3386  *   Pointer to flow attributes
3387  * @param[out] error
3388  *   Pointer to error structure.
3389  *
3390  * @return
3391  *   0 on success, a negative errno value otherwise and rte_errno is set.
3392  */
3393 static int
3394 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
3395                               uint64_t action_flags,
3396                               const struct rte_flow_action *action,
3397                               const uint64_t item_flags,
3398                               const struct rte_flow_attr *attr,
3399                               struct rte_flow_error *error)
3400 {
3401         const struct mlx5_priv *priv = dev->data->dev_private;
3402
3403         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
3404             !priv->config.decap_en)
3405                 return rte_flow_error_set(error, ENOTSUP,
3406                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3407                                           "decap is not enabled");
3408         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
3409                 return rte_flow_error_set(error, ENOTSUP,
3410                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3411                                           action_flags &
3412                                           MLX5_FLOW_ACTION_DECAP ? "can only "
3413                                           "have a single decap action" : "decap "
3414                                           "after encap is not supported");
3415         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
3416                 return rte_flow_error_set(error, EINVAL,
3417                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3418                                           "can't have decap action after"
3419                                           " modify action");
3420         if (attr->egress)
3421                 return rte_flow_error_set(error, ENOTSUP,
3422                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
3423                                           NULL,
3424                                           "decap action not supported for "
3425                                           "egress");
3426         if (!attr->transfer && priv->representor)
3427                 return rte_flow_error_set(error, ENOTSUP,
3428                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3429                                           "decap action for VF representor "
3430                                           "not supported on NIC table");
3431         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
3432             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
3433                 return rte_flow_error_set(error, ENOTSUP,
3434                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3435                                 "VXLAN item should be present for VXLAN decap");
3436         return 0;
3437 }
3438
3439 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
3440
3441 /**
3442  * Validate the raw encap and decap actions.
3443  *
3444  * @param[in] dev
3445  *   Pointer to the rte_eth_dev structure.
3446  * @param[in] decap
3447  *   Pointer to the decap action.
3448  * @param[in] encap
3449  *   Pointer to the encap action.
3450  * @param[in] attr
3451  *   Pointer to flow attributes
3452  * @param[in/out] action_flags
3453  *   Holds the actions detected until now.
3454  * @param[out] actions_n
3455  *   pointer to the number of actions counter.
3456  * @param[in] action
3457  *   Pointer to the action structure.
3458  * @param[in] item_flags
3459  *   Holds the items detected.
3460  * @param[out] error
3461  *   Pointer to error structure.
3462  *
3463  * @return
3464  *   0 on success, a negative errno value otherwise and rte_errno is set.
3465  */
3466 static int
3467 flow_dv_validate_action_raw_encap_decap
3468         (struct rte_eth_dev *dev,
3469          const struct rte_flow_action_raw_decap *decap,
3470          const struct rte_flow_action_raw_encap *encap,
3471          const struct rte_flow_attr *attr, uint64_t *action_flags,
3472          int *actions_n, const struct rte_flow_action *action,
3473          uint64_t item_flags, struct rte_flow_error *error)
3474 {
3475         const struct mlx5_priv *priv = dev->data->dev_private;
3476         int ret;
3477
3478         if (encap && (!encap->size || !encap->data))
3479                 return rte_flow_error_set(error, EINVAL,
3480                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3481                                           "raw encap data cannot be empty");
3482         if (decap && encap) {
3483                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
3484                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3485                         /* L3 encap. */
3486                         decap = NULL;
3487                 else if (encap->size <=
3488                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3489                            decap->size >
3490                            MLX5_ENCAPSULATION_DECISION_SIZE)
3491                         /* L3 decap. */
3492                         encap = NULL;
3493                 else if (encap->size >
3494                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3495                            decap->size >
3496                            MLX5_ENCAPSULATION_DECISION_SIZE)
3497                         /* 2 L2 actions: encap and decap. */
3498                         ;
3499                 else
3500                         return rte_flow_error_set(error,
3501                                 ENOTSUP,
3502                                 RTE_FLOW_ERROR_TYPE_ACTION,
3503                                 NULL, "unsupported too small "
3504                                 "raw decap and too small raw "
3505                                 "encap combination");
3506         }
3507         if (decap) {
3508                 ret = flow_dv_validate_action_decap(dev, *action_flags, action,
3509                                                     item_flags, attr, error);
3510                 if (ret < 0)
3511                         return ret;
3512                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
3513                 ++(*actions_n);
3514         }
3515         if (encap) {
3516                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
3517                         return rte_flow_error_set(error, ENOTSUP,
3518                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3519                                                   NULL,
3520                                                   "small raw encap size");
3521                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
3522                         return rte_flow_error_set(error, EINVAL,
3523                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3524                                                   NULL,
3525                                                   "more than one encap action");
3526                 if (!attr->transfer && priv->representor)
3527                         return rte_flow_error_set
3528                                         (error, ENOTSUP,
3529                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3530                                          "encap action for VF representor "
3531                                          "not supported on NIC table");
3532                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
3533                 ++(*actions_n);
3534         }
3535         return 0;
3536 }
3537
3538 /*
3539  * Validate the ASO CT action.
3540  *
3541  * @param[in] dev
3542  *   Pointer to the rte_eth_dev structure.
3543  * @param[in] action_flags
3544  *   Holds the actions detected until now.
3545  * @param[in] item_flags
3546  *   The items found in this flow rule.
3547  * @param[in] attr
3548  *   Pointer to flow attributes.
3549  * @param[out] error
3550  *   Pointer to error structure.
3551  *
3552  * @return
3553  *   0 on success, a negative errno value otherwise and rte_errno is set.
3554  */
3555 static int
3556 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
3557                                uint64_t action_flags,
3558                                uint64_t item_flags,
3559                                const struct rte_flow_attr *attr,
3560                                struct rte_flow_error *error)
3561 {
3562         RTE_SET_USED(dev);
3563
3564         if (attr->group == 0 && !attr->transfer)
3565                 return rte_flow_error_set(error, ENOTSUP,
3566                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3567                                           NULL,
3568                                           "Only support non-root table");
3569         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
3570                 return rte_flow_error_set(error, ENOTSUP,
3571                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3572                                           "CT cannot follow a fate action");
3573         if ((action_flags & MLX5_FLOW_ACTION_METER) ||
3574             (action_flags & MLX5_FLOW_ACTION_AGE))
3575                 return rte_flow_error_set(error, EINVAL,
3576                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3577                                           "Only one ASO action is supported");
3578         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3579                 return rte_flow_error_set(error, EINVAL,
3580                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3581                                           "Encap cannot exist before CT");
3582         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3583                 return rte_flow_error_set(error, EINVAL,
3584                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3585                                           "Not a outer TCP packet");
3586         return 0;
3587 }
3588
3589 int
3590 flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
3591                              struct mlx5_list_entry *entry, void *cb_ctx)
3592 {
3593         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3594         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3595         struct mlx5_flow_dv_encap_decap_resource *resource;
3596
3597         resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
3598                                 entry);
3599         if (resource->reformat_type == ctx_resource->reformat_type &&
3600             resource->ft_type == ctx_resource->ft_type &&
3601             resource->flags == ctx_resource->flags &&
3602             resource->size == ctx_resource->size &&
3603             !memcmp((const void *)resource->buf,
3604                     (const void *)ctx_resource->buf,
3605                     resource->size))
3606                 return 0;
3607         return -1;
3608 }
3609
3610 struct mlx5_list_entry *
3611 flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
3612 {
3613         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3614         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3615         struct mlx5dv_dr_domain *domain;
3616         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3617         struct mlx5_flow_dv_encap_decap_resource *resource;
3618         uint32_t idx;
3619         int ret;
3620
3621         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3622                 domain = sh->fdb_domain;
3623         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3624                 domain = sh->rx_domain;
3625         else
3626                 domain = sh->tx_domain;
3627         /* Register new encap/decap resource. */
3628         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
3629         if (!resource) {
3630                 rte_flow_error_set(ctx->error, ENOMEM,
3631                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3632                                    "cannot allocate resource memory");
3633                 return NULL;
3634         }
3635         *resource = *ctx_resource;
3636         resource->idx = idx;
3637         ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain,
3638                                                               resource,
3639                                                              &resource->action);
3640         if (ret) {
3641                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
3642                 rte_flow_error_set(ctx->error, ENOMEM,
3643                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3644                                    NULL, "cannot create action");
3645                 return NULL;
3646         }
3647
3648         return &resource->entry;
3649 }
3650
3651 struct mlx5_list_entry *
3652 flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
3653                              void *cb_ctx)
3654 {
3655         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3656         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3657         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3658         uint32_t idx;
3659
3660         cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
3661                                            &idx);
3662         if (!cache_resource) {
3663                 rte_flow_error_set(ctx->error, ENOMEM,
3664                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3665                                    "cannot allocate resource memory");
3666                 return NULL;
3667         }
3668         memcpy(cache_resource, oentry, sizeof(*cache_resource));
3669         cache_resource->idx = idx;
3670         return &cache_resource->entry;
3671 }
3672
3673 void
3674 flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3675 {
3676         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3677         struct mlx5_flow_dv_encap_decap_resource *res =
3678                                        container_of(entry, typeof(*res), entry);
3679
3680         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
3681 }
3682
3683 /**
3684  * Find existing encap/decap resource or create and register a new one.
3685  *
3686  * @param[in, out] dev
3687  *   Pointer to rte_eth_dev structure.
3688  * @param[in, out] resource
3689  *   Pointer to encap/decap resource.
3690  * @parm[in, out] dev_flow
3691  *   Pointer to the dev_flow.
3692  * @param[out] error
3693  *   pointer to error structure.
3694  *
3695  * @return
3696  *   0 on success otherwise -errno and errno is set.
3697  */
3698 static int
3699 flow_dv_encap_decap_resource_register
3700                         (struct rte_eth_dev *dev,
3701                          struct mlx5_flow_dv_encap_decap_resource *resource,
3702                          struct mlx5_flow *dev_flow,
3703                          struct rte_flow_error *error)
3704 {
3705         struct mlx5_priv *priv = dev->data->dev_private;
3706         struct mlx5_dev_ctx_shared *sh = priv->sh;
3707         struct mlx5_list_entry *entry;
3708         union {
3709                 struct {
3710                         uint32_t ft_type:8;
3711                         uint32_t refmt_type:8;
3712                         /*
3713                          * Header reformat actions can be shared between
3714                          * non-root tables. One bit to indicate non-root
3715                          * table or not.
3716                          */
3717                         uint32_t is_root:1;
3718                         uint32_t reserve:15;
3719                 };
3720                 uint32_t v32;
3721         } encap_decap_key = {
3722                 {
3723                         .ft_type = resource->ft_type,
3724                         .refmt_type = resource->reformat_type,
3725                         .is_root = !!dev_flow->dv.group,
3726                         .reserve = 0,
3727                 }
3728         };
3729         struct mlx5_flow_cb_ctx ctx = {
3730                 .error = error,
3731                 .data = resource,
3732         };
3733         uint64_t key64;
3734
3735         resource->flags = dev_flow->dv.group ? 0 : 1;
3736         key64 =  __rte_raw_cksum(&encap_decap_key.v32,
3737                                  sizeof(encap_decap_key.v32), 0);
3738         if (resource->reformat_type !=
3739             MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
3740             resource->size)
3741                 key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
3742         entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
3743         if (!entry)
3744                 return -rte_errno;
3745         resource = container_of(entry, typeof(*resource), entry);
3746         dev_flow->dv.encap_decap = resource;
3747         dev_flow->handle->dvh.rix_encap_decap = resource->idx;
3748         return 0;
3749 }
3750
3751 /**
3752  * Find existing table jump resource or create and register a new one.
3753  *
3754  * @param[in, out] dev
3755  *   Pointer to rte_eth_dev structure.
3756  * @param[in, out] tbl
3757  *   Pointer to flow table resource.
3758  * @parm[in, out] dev_flow
3759  *   Pointer to the dev_flow.
3760  * @param[out] error
3761  *   pointer to error structure.
3762  *
3763  * @return
3764  *   0 on success otherwise -errno and errno is set.
3765  */
3766 static int
3767 flow_dv_jump_tbl_resource_register
3768                         (struct rte_eth_dev *dev __rte_unused,
3769                          struct mlx5_flow_tbl_resource *tbl,
3770                          struct mlx5_flow *dev_flow,
3771                          struct rte_flow_error *error __rte_unused)
3772 {
3773         struct mlx5_flow_tbl_data_entry *tbl_data =
3774                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
3775
3776         MLX5_ASSERT(tbl);
3777         MLX5_ASSERT(tbl_data->jump.action);
3778         dev_flow->handle->rix_jump = tbl_data->idx;
3779         dev_flow->dv.jump = &tbl_data->jump;
3780         return 0;
3781 }
3782
3783 int
3784 flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
3785                          struct mlx5_list_entry *entry, void *cb_ctx)
3786 {
3787         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3788         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3789         struct mlx5_flow_dv_port_id_action_resource *res =
3790                                        container_of(entry, typeof(*res), entry);
3791
3792         return ref->port_id != res->port_id;
3793 }
3794
3795 struct mlx5_list_entry *
3796 flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
3797 {
3798         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3799         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3800         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3801         struct mlx5_flow_dv_port_id_action_resource *resource;
3802         uint32_t idx;
3803         int ret;
3804
3805         /* Register new port id action resource. */
3806         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3807         if (!resource) {
3808                 rte_flow_error_set(ctx->error, ENOMEM,
3809                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3810                                    "cannot allocate port_id action memory");
3811                 return NULL;
3812         }
3813         *resource = *ref;
3814         ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
3815                                                         ref->port_id,
3816                                                         &resource->action);
3817         if (ret) {
3818                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
3819                 rte_flow_error_set(ctx->error, ENOMEM,
3820                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3821                                    "cannot create action");
3822                 return NULL;
3823         }
3824         resource->idx = idx;
3825         return &resource->entry;
3826 }
3827
3828 struct mlx5_list_entry *
3829 flow_dv_port_id_clone_cb(void *tool_ctx,
3830                          struct mlx5_list_entry *entry __rte_unused,
3831                          void *cb_ctx)
3832 {
3833         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3834         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3835         struct mlx5_flow_dv_port_id_action_resource *resource;
3836         uint32_t idx;
3837
3838         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3839         if (!resource) {
3840                 rte_flow_error_set(ctx->error, ENOMEM,
3841                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3842                                    "cannot allocate port_id action memory");
3843                 return NULL;
3844         }
3845         memcpy(resource, entry, sizeof(*resource));
3846         resource->idx = idx;
3847         return &resource->entry;
3848 }
3849
3850 void
3851 flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3852 {
3853         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3854         struct mlx5_flow_dv_port_id_action_resource *resource =
3855                                   container_of(entry, typeof(*resource), entry);
3856
3857         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
3858 }
3859
3860 /**
3861  * Find existing table port ID resource or create and register a new one.
3862  *
3863  * @param[in, out] dev
3864  *   Pointer to rte_eth_dev structure.
3865  * @param[in, out] ref
3866  *   Pointer to port ID action resource reference.
3867  * @parm[in, out] dev_flow
3868  *   Pointer to the dev_flow.
3869  * @param[out] error
3870  *   pointer to error structure.
3871  *
3872  * @return
3873  *   0 on success otherwise -errno and errno is set.
3874  */
3875 static int
3876 flow_dv_port_id_action_resource_register
3877                         (struct rte_eth_dev *dev,
3878                          struct mlx5_flow_dv_port_id_action_resource *ref,
3879                          struct mlx5_flow *dev_flow,
3880                          struct rte_flow_error *error)
3881 {
3882         struct mlx5_priv *priv = dev->data->dev_private;
3883         struct mlx5_list_entry *entry;
3884         struct mlx5_flow_dv_port_id_action_resource *resource;
3885         struct mlx5_flow_cb_ctx ctx = {
3886                 .error = error,
3887                 .data = ref,
3888         };
3889
3890         entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
3891         if (!entry)
3892                 return -rte_errno;
3893         resource = container_of(entry, typeof(*resource), entry);
3894         dev_flow->dv.port_id_action = resource;
3895         dev_flow->handle->rix_port_id_action = resource->idx;
3896         return 0;
3897 }
3898
3899 int
3900 flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
3901                            struct mlx5_list_entry *entry, void *cb_ctx)
3902 {
3903         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3904         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3905         struct mlx5_flow_dv_push_vlan_action_resource *res =
3906                                        container_of(entry, typeof(*res), entry);
3907
3908         return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
3909 }
3910
3911 struct mlx5_list_entry *
3912 flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
3913 {
3914         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3915         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3916         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3917         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3918         struct mlx5dv_dr_domain *domain;
3919         uint32_t idx;
3920         int ret;
3921
3922         /* Register new port id action resource. */
3923         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3924         if (!resource) {
3925                 rte_flow_error_set(ctx->error, ENOMEM,
3926                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3927                                    "cannot allocate push_vlan action memory");
3928                 return NULL;
3929         }
3930         *resource = *ref;
3931         if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3932                 domain = sh->fdb_domain;
3933         else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3934                 domain = sh->rx_domain;
3935         else
3936                 domain = sh->tx_domain;
3937         ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
3938                                                         &resource->action);
3939         if (ret) {
3940                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
3941                 rte_flow_error_set(ctx->error, ENOMEM,
3942                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3943                                    "cannot create push vlan action");
3944                 return NULL;
3945         }
3946         resource->idx = idx;
3947         return &resource->entry;
3948 }
3949
3950 struct mlx5_list_entry *
3951 flow_dv_push_vlan_clone_cb(void *tool_ctx,
3952                            struct mlx5_list_entry *entry __rte_unused,
3953                            void *cb_ctx)
3954 {
3955         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3956         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3957         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3958         uint32_t idx;
3959
3960         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3961         if (!resource) {
3962                 rte_flow_error_set(ctx->error, ENOMEM,
3963                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3964                                    "cannot allocate push_vlan action memory");
3965                 return NULL;
3966         }
3967         memcpy(resource, entry, sizeof(*resource));
3968         resource->idx = idx;
3969         return &resource->entry;
3970 }
3971
3972 void
3973 flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3974 {
3975         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3976         struct mlx5_flow_dv_push_vlan_action_resource *resource =
3977                                   container_of(entry, typeof(*resource), entry);
3978
3979         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
3980 }
3981
3982 /**
3983  * Find existing push vlan resource or create and register a new one.
3984  *
3985  * @param [in, out] dev
3986  *   Pointer to rte_eth_dev structure.
3987  * @param[in, out] ref
3988  *   Pointer to port ID action resource reference.
3989  * @parm[in, out] dev_flow
3990  *   Pointer to the dev_flow.
3991  * @param[out] error
3992  *   pointer to error structure.
3993  *
3994  * @return
3995  *   0 on success otherwise -errno and errno is set.
3996  */
3997 static int
3998 flow_dv_push_vlan_action_resource_register
3999                        (struct rte_eth_dev *dev,
4000                         struct mlx5_flow_dv_push_vlan_action_resource *ref,
4001                         struct mlx5_flow *dev_flow,
4002                         struct rte_flow_error *error)
4003 {
4004         struct mlx5_priv *priv = dev->data->dev_private;
4005         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4006         struct mlx5_list_entry *entry;
4007         struct mlx5_flow_cb_ctx ctx = {
4008                 .error = error,
4009                 .data = ref,
4010         };
4011
4012         entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
4013         if (!entry)
4014                 return -rte_errno;
4015         resource = container_of(entry, typeof(*resource), entry);
4016
4017         dev_flow->handle->dvh.rix_push_vlan = resource->idx;
4018         dev_flow->dv.push_vlan_res = resource;
4019         return 0;
4020 }
4021
4022 /**
4023  * Get the size of specific rte_flow_item_type hdr size
4024  *
4025  * @param[in] item_type
4026  *   Tested rte_flow_item_type.
4027  *
4028  * @return
4029  *   sizeof struct item_type, 0 if void or irrelevant.
4030  */
4031 static size_t
4032 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
4033 {
4034         size_t retval;
4035
4036         switch (item_type) {
4037         case RTE_FLOW_ITEM_TYPE_ETH:
4038                 retval = sizeof(struct rte_ether_hdr);
4039                 break;
4040         case RTE_FLOW_ITEM_TYPE_VLAN:
4041                 retval = sizeof(struct rte_vlan_hdr);
4042                 break;
4043         case RTE_FLOW_ITEM_TYPE_IPV4:
4044                 retval = sizeof(struct rte_ipv4_hdr);
4045                 break;
4046         case RTE_FLOW_ITEM_TYPE_IPV6:
4047                 retval = sizeof(struct rte_ipv6_hdr);
4048                 break;
4049         case RTE_FLOW_ITEM_TYPE_UDP:
4050                 retval = sizeof(struct rte_udp_hdr);
4051                 break;
4052         case RTE_FLOW_ITEM_TYPE_TCP:
4053                 retval = sizeof(struct rte_tcp_hdr);
4054                 break;
4055         case RTE_FLOW_ITEM_TYPE_VXLAN:
4056         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4057                 retval = sizeof(struct rte_vxlan_hdr);
4058                 break;
4059         case RTE_FLOW_ITEM_TYPE_GRE:
4060         case RTE_FLOW_ITEM_TYPE_NVGRE:
4061                 retval = sizeof(struct rte_gre_hdr);
4062                 break;
4063         case RTE_FLOW_ITEM_TYPE_MPLS:
4064                 retval = sizeof(struct rte_mpls_hdr);
4065                 break;
4066         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
4067         default:
4068                 retval = 0;
4069                 break;
4070         }
4071         return retval;
4072 }
4073
4074 #define MLX5_ENCAP_IPV4_VERSION         0x40
4075 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
4076 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
4077 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
4078 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
4079 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
4080 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
4081
4082 /**
4083  * Convert the encap action data from list of rte_flow_item to raw buffer
4084  *
4085  * @param[in] items
4086  *   Pointer to rte_flow_item objects list.
4087  * @param[out] buf
4088  *   Pointer to the output buffer.
4089  * @param[out] size
4090  *   Pointer to the output buffer size.
4091  * @param[out] error
4092  *   Pointer to the error structure.
4093  *
4094  * @return
4095  *   0 on success, a negative errno value otherwise and rte_errno is set.
4096  */
4097 static int
4098 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
4099                            size_t *size, struct rte_flow_error *error)
4100 {
4101         struct rte_ether_hdr *eth = NULL;
4102         struct rte_vlan_hdr *vlan = NULL;
4103         struct rte_ipv4_hdr *ipv4 = NULL;
4104         struct rte_ipv6_hdr *ipv6 = NULL;
4105         struct rte_udp_hdr *udp = NULL;
4106         struct rte_vxlan_hdr *vxlan = NULL;
4107         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
4108         struct rte_gre_hdr *gre = NULL;
4109         size_t len;
4110         size_t temp_size = 0;
4111
4112         if (!items)
4113                 return rte_flow_error_set(error, EINVAL,
4114                                           RTE_FLOW_ERROR_TYPE_ACTION,
4115                                           NULL, "invalid empty data");
4116         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4117                 len = flow_dv_get_item_hdr_len(items->type);
4118                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
4119                         return rte_flow_error_set(error, EINVAL,
4120                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4121                                                   (void *)items->type,
4122                                                   "items total size is too big"
4123                                                   " for encap action");
4124                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
4125                 switch (items->type) {
4126                 case RTE_FLOW_ITEM_TYPE_ETH:
4127                         eth = (struct rte_ether_hdr *)&buf[temp_size];
4128                         break;
4129                 case RTE_FLOW_ITEM_TYPE_VLAN:
4130                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
4131                         if (!eth)
4132                                 return rte_flow_error_set(error, EINVAL,
4133                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4134                                                 (void *)items->type,
4135                                                 "eth header not found");
4136                         if (!eth->ether_type)
4137                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
4138                         break;
4139                 case RTE_FLOW_ITEM_TYPE_IPV4:
4140                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
4141                         if (!vlan && !eth)
4142                                 return rte_flow_error_set(error, EINVAL,
4143                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4144                                                 (void *)items->type,
4145                                                 "neither eth nor vlan"
4146                                                 " header found");
4147                         if (vlan && !vlan->eth_proto)
4148                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4149                         else if (eth && !eth->ether_type)
4150                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4151                         if (!ipv4->version_ihl)
4152                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
4153                                                     MLX5_ENCAP_IPV4_IHL_MIN;
4154                         if (!ipv4->time_to_live)
4155                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
4156                         break;
4157                 case RTE_FLOW_ITEM_TYPE_IPV6:
4158                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
4159                         if (!vlan && !eth)
4160                                 return rte_flow_error_set(error, EINVAL,
4161                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4162                                                 (void *)items->type,
4163                                                 "neither eth nor vlan"
4164                                                 " header found");
4165                         if (vlan && !vlan->eth_proto)
4166                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4167                         else if (eth && !eth->ether_type)
4168                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4169                         if (!ipv6->vtc_flow)
4170                                 ipv6->vtc_flow =
4171                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
4172                         if (!ipv6->hop_limits)
4173                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
4174                         break;
4175                 case RTE_FLOW_ITEM_TYPE_UDP:
4176                         udp = (struct rte_udp_hdr *)&buf[temp_size];
4177                         if (!ipv4 && !ipv6)
4178                                 return rte_flow_error_set(error, EINVAL,
4179                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4180                                                 (void *)items->type,
4181                                                 "ip header not found");
4182                         if (ipv4 && !ipv4->next_proto_id)
4183                                 ipv4->next_proto_id = IPPROTO_UDP;
4184                         else if (ipv6 && !ipv6->proto)
4185                                 ipv6->proto = IPPROTO_UDP;
4186                         break;
4187                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4188                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
4189                         if (!udp)
4190                                 return rte_flow_error_set(error, EINVAL,
4191                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4192                                                 (void *)items->type,
4193                                                 "udp header not found");
4194                         if (!udp->dst_port)
4195                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
4196                         if (!vxlan->vx_flags)
4197                                 vxlan->vx_flags =
4198                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
4199                         break;
4200                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4201                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
4202                         if (!udp)
4203                                 return rte_flow_error_set(error, EINVAL,
4204                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4205                                                 (void *)items->type,
4206                                                 "udp header not found");
4207                         if (!vxlan_gpe->proto)
4208                                 return rte_flow_error_set(error, EINVAL,
4209                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4210                                                 (void *)items->type,
4211                                                 "next protocol not found");
4212                         if (!udp->dst_port)
4213                                 udp->dst_port =
4214                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
4215                         if (!vxlan_gpe->vx_flags)
4216                                 vxlan_gpe->vx_flags =
4217                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
4218                         break;
4219                 case RTE_FLOW_ITEM_TYPE_GRE:
4220                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4221                         gre = (struct rte_gre_hdr *)&buf[temp_size];
4222                         if (!gre->proto)
4223                                 return rte_flow_error_set(error, EINVAL,
4224                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4225                                                 (void *)items->type,
4226                                                 "next protocol not found");
4227                         if (!ipv4 && !ipv6)
4228                                 return rte_flow_error_set(error, EINVAL,
4229                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4230                                                 (void *)items->type,
4231                                                 "ip header not found");
4232                         if (ipv4 && !ipv4->next_proto_id)
4233                                 ipv4->next_proto_id = IPPROTO_GRE;
4234                         else if (ipv6 && !ipv6->proto)
4235                                 ipv6->proto = IPPROTO_GRE;
4236                         break;
4237                 case RTE_FLOW_ITEM_TYPE_VOID:
4238                         break;
4239                 default:
4240                         return rte_flow_error_set(error, EINVAL,
4241                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4242                                                   (void *)items->type,
4243                                                   "unsupported item type");
4244                         break;
4245                 }
4246                 temp_size += len;
4247         }
4248         *size = temp_size;
4249         return 0;
4250 }
4251
4252 static int
4253 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
4254 {
4255         struct rte_ether_hdr *eth = NULL;
4256         struct rte_vlan_hdr *vlan = NULL;
4257         struct rte_ipv6_hdr *ipv6 = NULL;
4258         struct rte_udp_hdr *udp = NULL;
4259         char *next_hdr;
4260         uint16_t proto;
4261
4262         eth = (struct rte_ether_hdr *)data;
4263         next_hdr = (char *)(eth + 1);
4264         proto = RTE_BE16(eth->ether_type);
4265
4266         /* VLAN skipping */
4267         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
4268                 vlan = (struct rte_vlan_hdr *)next_hdr;
4269                 proto = RTE_BE16(vlan->eth_proto);
4270                 next_hdr += sizeof(struct rte_vlan_hdr);
4271         }
4272
4273         /* HW calculates IPv4 csum. no need to proceed */
4274         if (proto == RTE_ETHER_TYPE_IPV4)
4275                 return 0;
4276
4277         /* non IPv4/IPv6 header. not supported */
4278         if (proto != RTE_ETHER_TYPE_IPV6) {
4279                 return rte_flow_error_set(error, ENOTSUP,
4280                                           RTE_FLOW_ERROR_TYPE_ACTION,
4281                                           NULL, "Cannot offload non IPv4/IPv6");
4282         }
4283
4284         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
4285
4286         /* ignore non UDP */
4287         if (ipv6->proto != IPPROTO_UDP)
4288                 return 0;
4289
4290         udp = (struct rte_udp_hdr *)(ipv6 + 1);
4291         udp->dgram_cksum = 0;
4292
4293         return 0;
4294 }
4295
4296 /**
4297  * Convert L2 encap action to DV specification.
4298  *
4299  * @param[in] dev
4300  *   Pointer to rte_eth_dev structure.
4301  * @param[in] action
4302  *   Pointer to action structure.
4303  * @param[in, out] dev_flow
4304  *   Pointer to the mlx5_flow.
4305  * @param[in] transfer
4306  *   Mark if the flow is E-Switch flow.
4307  * @param[out] error
4308  *   Pointer to the error structure.
4309  *
4310  * @return
4311  *   0 on success, a negative errno value otherwise and rte_errno is set.
4312  */
4313 static int
4314 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
4315                                const struct rte_flow_action *action,
4316                                struct mlx5_flow *dev_flow,
4317                                uint8_t transfer,
4318                                struct rte_flow_error *error)
4319 {
4320         const struct rte_flow_item *encap_data;
4321         const struct rte_flow_action_raw_encap *raw_encap_data;
4322         struct mlx5_flow_dv_encap_decap_resource res = {
4323                 .reformat_type =
4324                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
4325                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4326                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
4327         };
4328
4329         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4330                 raw_encap_data =
4331                         (const struct rte_flow_action_raw_encap *)action->conf;
4332                 res.size = raw_encap_data->size;
4333                 memcpy(res.buf, raw_encap_data->data, res.size);
4334         } else {
4335                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
4336                         encap_data =
4337                                 ((const struct rte_flow_action_vxlan_encap *)
4338                                                 action->conf)->definition;
4339                 else
4340                         encap_data =
4341                                 ((const struct rte_flow_action_nvgre_encap *)
4342                                                 action->conf)->definition;
4343                 if (flow_dv_convert_encap_data(encap_data, res.buf,
4344                                                &res.size, error))
4345                         return -rte_errno;
4346         }
4347         if (flow_dv_zero_encap_udp_csum(res.buf, error))
4348                 return -rte_errno;
4349         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4350                 return rte_flow_error_set(error, EINVAL,
4351                                           RTE_FLOW_ERROR_TYPE_ACTION,
4352                                           NULL, "can't create L2 encap action");
4353         return 0;
4354 }
4355
4356 /**
4357  * Convert L2 decap action to DV specification.
4358  *
4359  * @param[in] dev
4360  *   Pointer to rte_eth_dev structure.
4361  * @param[in, out] dev_flow
4362  *   Pointer to the mlx5_flow.
4363  * @param[in] transfer
4364  *   Mark if the flow is E-Switch flow.
4365  * @param[out] error
4366  *   Pointer to the error structure.
4367  *
4368  * @return
4369  *   0 on success, a negative errno value otherwise and rte_errno is set.
4370  */
4371 static int
4372 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
4373                                struct mlx5_flow *dev_flow,
4374                                uint8_t transfer,
4375                                struct rte_flow_error *error)
4376 {
4377         struct mlx5_flow_dv_encap_decap_resource res = {
4378                 .size = 0,
4379                 .reformat_type =
4380                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
4381                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4382                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
4383         };
4384
4385         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4386                 return rte_flow_error_set(error, EINVAL,
4387                                           RTE_FLOW_ERROR_TYPE_ACTION,
4388                                           NULL, "can't create L2 decap action");
4389         return 0;
4390 }
4391
4392 /**
4393  * Convert raw decap/encap (L3 tunnel) action to DV specification.
4394  *
4395  * @param[in] dev
4396  *   Pointer to rte_eth_dev structure.
4397  * @param[in] action
4398  *   Pointer to action structure.
4399  * @param[in, out] dev_flow
4400  *   Pointer to the mlx5_flow.
4401  * @param[in] attr
4402  *   Pointer to the flow attributes.
4403  * @param[out] error
4404  *   Pointer to the error structure.
4405  *
4406  * @return
4407  *   0 on success, a negative errno value otherwise and rte_errno is set.
4408  */
4409 static int
4410 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
4411                                 const struct rte_flow_action *action,
4412                                 struct mlx5_flow *dev_flow,
4413                                 const struct rte_flow_attr *attr,
4414                                 struct rte_flow_error *error)
4415 {
4416         const struct rte_flow_action_raw_encap *encap_data;
4417         struct mlx5_flow_dv_encap_decap_resource res;
4418
4419         memset(&res, 0, sizeof(res));
4420         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
4421         res.size = encap_data->size;
4422         memcpy(res.buf, encap_data->data, res.size);
4423         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
4424                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
4425                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
4426         if (attr->transfer)
4427                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4428         else
4429                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4430                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4431         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4432                 return rte_flow_error_set(error, EINVAL,
4433                                           RTE_FLOW_ERROR_TYPE_ACTION,
4434                                           NULL, "can't create encap action");
4435         return 0;
4436 }
4437
4438 /**
4439  * Create action push VLAN.
4440  *
4441  * @param[in] dev
4442  *   Pointer to rte_eth_dev structure.
4443  * @param[in] attr
4444  *   Pointer to the flow attributes.
4445  * @param[in] vlan
4446  *   Pointer to the vlan to push to the Ethernet header.
4447  * @param[in, out] dev_flow
4448  *   Pointer to the mlx5_flow.
4449  * @param[out] error
4450  *   Pointer to the error structure.
4451  *
4452  * @return
4453  *   0 on success, a negative errno value otherwise and rte_errno is set.
4454  */
4455 static int
4456 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
4457                                 const struct rte_flow_attr *attr,
4458                                 const struct rte_vlan_hdr *vlan,
4459                                 struct mlx5_flow *dev_flow,
4460                                 struct rte_flow_error *error)
4461 {
4462         struct mlx5_flow_dv_push_vlan_action_resource res;
4463
4464         memset(&res, 0, sizeof(res));
4465         res.vlan_tag =
4466                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
4467                                  vlan->vlan_tci);
4468         if (attr->transfer)
4469                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4470         else
4471                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4472                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4473         return flow_dv_push_vlan_action_resource_register
4474                                             (dev, &res, dev_flow, error);
4475 }
4476
4477 /**
4478  * Validate the modify-header actions.
4479  *
4480  * @param[in] action_flags
4481  *   Holds the actions detected until now.
4482  * @param[in] action
4483  *   Pointer to the modify action.
4484  * @param[out] error
4485  *   Pointer to error structure.
4486  *
4487  * @return
4488  *   0 on success, a negative errno value otherwise and rte_errno is set.
4489  */
4490 static int
4491 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
4492                                    const struct rte_flow_action *action,
4493                                    struct rte_flow_error *error)
4494 {
4495         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
4496                 return rte_flow_error_set(error, EINVAL,
4497                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4498                                           NULL, "action configuration not set");
4499         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
4500                 return rte_flow_error_set(error, EINVAL,
4501                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4502                                           "can't have encap action before"
4503                                           " modify action");
4504         return 0;
4505 }
4506
4507 /**
4508  * Validate the modify-header MAC address actions.
4509  *
4510  * @param[in] action_flags
4511  *   Holds the actions detected until now.
4512  * @param[in] action
4513  *   Pointer to the modify action.
4514  * @param[in] item_flags
4515  *   Holds the items detected.
4516  * @param[out] error
4517  *   Pointer to error structure.
4518  *
4519  * @return
4520  *   0 on success, a negative errno value otherwise and rte_errno is set.
4521  */
4522 static int
4523 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
4524                                    const struct rte_flow_action *action,
4525                                    const uint64_t item_flags,
4526                                    struct rte_flow_error *error)
4527 {
4528         int ret = 0;
4529
4530         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4531         if (!ret) {
4532                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
4533                         return rte_flow_error_set(error, EINVAL,
4534                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4535                                                   NULL,
4536                                                   "no L2 item in pattern");
4537         }
4538         return ret;
4539 }
4540
4541 /**
4542  * Validate the modify-header IPv4 address actions.
4543  *
4544  * @param[in] action_flags
4545  *   Holds the actions detected until now.
4546  * @param[in] action
4547  *   Pointer to the modify action.
4548  * @param[in] item_flags
4549  *   Holds the items detected.
4550  * @param[out] error
4551  *   Pointer to error structure.
4552  *
4553  * @return
4554  *   0 on success, a negative errno value otherwise and rte_errno is set.
4555  */
4556 static int
4557 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
4558                                     const struct rte_flow_action *action,
4559                                     const uint64_t item_flags,
4560                                     struct rte_flow_error *error)
4561 {
4562         int ret = 0;
4563         uint64_t layer;
4564
4565         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4566         if (!ret) {
4567                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4568                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4569                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4570                 if (!(item_flags & layer))
4571                         return rte_flow_error_set(error, EINVAL,
4572                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4573                                                   NULL,
4574                                                   "no ipv4 item in pattern");
4575         }
4576         return ret;
4577 }
4578
4579 /**
4580  * Validate the modify-header IPv6 address actions.
4581  *
4582  * @param[in] action_flags
4583  *   Holds the actions detected until now.
4584  * @param[in] action
4585  *   Pointer to the modify action.
4586  * @param[in] item_flags
4587  *   Holds the items detected.
4588  * @param[out] error
4589  *   Pointer to error structure.
4590  *
4591  * @return
4592  *   0 on success, a negative errno value otherwise and rte_errno is set.
4593  */
4594 static int
4595 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
4596                                     const struct rte_flow_action *action,
4597                                     const uint64_t item_flags,
4598                                     struct rte_flow_error *error)
4599 {
4600         int ret = 0;
4601         uint64_t layer;
4602
4603         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4604         if (!ret) {
4605                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4606                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4607                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4608                 if (!(item_flags & layer))
4609                         return rte_flow_error_set(error, EINVAL,
4610                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4611                                                   NULL,
4612                                                   "no ipv6 item in pattern");
4613         }
4614         return ret;
4615 }
4616
4617 /**
4618  * Validate the modify-header TP actions.
4619  *
4620  * @param[in] action_flags
4621  *   Holds the actions detected until now.
4622  * @param[in] action
4623  *   Pointer to the modify action.
4624  * @param[in] item_flags
4625  *   Holds the items detected.
4626  * @param[out] error
4627  *   Pointer to error structure.
4628  *
4629  * @return
4630  *   0 on success, a negative errno value otherwise and rte_errno is set.
4631  */
4632 static int
4633 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
4634                                   const struct rte_flow_action *action,
4635                                   const uint64_t item_flags,
4636                                   struct rte_flow_error *error)
4637 {
4638         int ret = 0;
4639         uint64_t layer;
4640
4641         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4642         if (!ret) {
4643                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4644                                  MLX5_FLOW_LAYER_INNER_L4 :
4645                                  MLX5_FLOW_LAYER_OUTER_L4;
4646                 if (!(item_flags & layer))
4647                         return rte_flow_error_set(error, EINVAL,
4648                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4649                                                   NULL, "no transport layer "
4650                                                   "in pattern");
4651         }
4652         return ret;
4653 }
4654
4655 /**
4656  * Validate the modify-header actions of increment/decrement
4657  * TCP Sequence-number.
4658  *
4659  * @param[in] action_flags
4660  *   Holds the actions detected until now.
4661  * @param[in] action
4662  *   Pointer to the modify action.
4663  * @param[in] item_flags
4664  *   Holds the items detected.
4665  * @param[out] error
4666  *   Pointer to error structure.
4667  *
4668  * @return
4669  *   0 on success, a negative errno value otherwise and rte_errno is set.
4670  */
4671 static int
4672 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
4673                                        const struct rte_flow_action *action,
4674                                        const uint64_t item_flags,
4675                                        struct rte_flow_error *error)
4676 {
4677         int ret = 0;
4678         uint64_t layer;
4679
4680         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4681         if (!ret) {
4682                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4683                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4684                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4685                 if (!(item_flags & layer))
4686                         return rte_flow_error_set(error, EINVAL,
4687                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4688                                                   NULL, "no TCP item in"
4689                                                   " pattern");
4690                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
4691                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
4692                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
4693                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
4694                         return rte_flow_error_set(error, EINVAL,
4695                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4696                                                   NULL,
4697                                                   "cannot decrease and increase"
4698                                                   " TCP sequence number"
4699                                                   " at the same time");
4700         }
4701         return ret;
4702 }
4703
4704 /**
4705  * Validate the modify-header actions of increment/decrement
4706  * TCP Acknowledgment number.
4707  *
4708  * @param[in] action_flags
4709  *   Holds the actions detected until now.
4710  * @param[in] action
4711  *   Pointer to the modify action.
4712  * @param[in] item_flags
4713  *   Holds the items detected.
4714  * @param[out] error
4715  *   Pointer to error structure.
4716  *
4717  * @return
4718  *   0 on success, a negative errno value otherwise and rte_errno is set.
4719  */
4720 static int
4721 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
4722                                        const struct rte_flow_action *action,
4723                                        const uint64_t item_flags,
4724                                        struct rte_flow_error *error)
4725 {
4726         int ret = 0;
4727         uint64_t layer;
4728
4729         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4730         if (!ret) {
4731                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4732                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4733                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4734                 if (!(item_flags & layer))
4735                         return rte_flow_error_set(error, EINVAL,
4736                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4737                                                   NULL, "no TCP item in"
4738                                                   " pattern");
4739                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
4740                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
4741                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
4742                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
4743                         return rte_flow_error_set(error, EINVAL,
4744                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4745                                                   NULL,
4746                                                   "cannot decrease and increase"
4747                                                   " TCP acknowledgment number"
4748                                                   " at the same time");
4749         }
4750         return ret;
4751 }
4752
4753 /**
4754  * Validate the modify-header TTL actions.
4755  *
4756  * @param[in] action_flags
4757  *   Holds the actions detected until now.
4758  * @param[in] action
4759  *   Pointer to the modify action.
4760  * @param[in] item_flags
4761  *   Holds the items detected.
4762  * @param[out] error
4763  *   Pointer to error structure.
4764  *
4765  * @return
4766  *   0 on success, a negative errno value otherwise and rte_errno is set.
4767  */
4768 static int
4769 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
4770                                    const struct rte_flow_action *action,
4771                                    const uint64_t item_flags,
4772                                    struct rte_flow_error *error)
4773 {
4774         int ret = 0;
4775         uint64_t layer;
4776
4777         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4778         if (!ret) {
4779                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4780                                  MLX5_FLOW_LAYER_INNER_L3 :
4781                                  MLX5_FLOW_LAYER_OUTER_L3;
4782                 if (!(item_flags & layer))
4783                         return rte_flow_error_set(error, EINVAL,
4784                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4785                                                   NULL,
4786                                                   "no IP protocol in pattern");
4787         }
4788         return ret;
4789 }
4790
4791 /**
4792  * Validate the generic modify field actions.
4793  * @param[in] dev
4794  *   Pointer to the rte_eth_dev structure.
4795  * @param[in] action_flags
4796  *   Holds the actions detected until now.
4797  * @param[in] action
4798  *   Pointer to the modify action.
4799  * @param[in] attr
4800  *   Pointer to the flow attributes.
4801  * @param[out] error
4802  *   Pointer to error structure.
4803  *
4804  * @return
4805  *   Number of header fields to modify (0 or more) on success,
4806  *   a negative errno value otherwise and rte_errno is set.
4807  */
4808 static int
4809 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev,
4810                                    const uint64_t action_flags,
4811                                    const struct rte_flow_action *action,
4812                                    const struct rte_flow_attr *attr,
4813                                    struct rte_flow_error *error)
4814 {
4815         int ret = 0;
4816         struct mlx5_priv *priv = dev->data->dev_private;
4817         struct mlx5_dev_config *config = &priv->config;
4818         const struct rte_flow_action_modify_field *action_modify_field =
4819                 action->conf;
4820         uint32_t dst_width = mlx5_flow_item_field_width(config,
4821                                 action_modify_field->dst.field);
4822         uint32_t src_width = mlx5_flow_item_field_width(config,
4823                                 action_modify_field->src.field);
4824
4825         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4826         if (ret)
4827                 return ret;
4828
4829         if (action_modify_field->width == 0)
4830                 return rte_flow_error_set(error, EINVAL,
4831                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4832                                 "no bits are requested to be modified");
4833         else if (action_modify_field->width > dst_width ||
4834                  action_modify_field->width > src_width)
4835                 return rte_flow_error_set(error, EINVAL,
4836                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4837                                 "cannot modify more bits than"
4838                                 " the width of a field");
4839         if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE &&
4840             action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) {
4841                 if ((action_modify_field->dst.offset +
4842                      action_modify_field->width > dst_width) ||
4843                     (action_modify_field->dst.offset % 32))
4844                         return rte_flow_error_set(error, EINVAL,
4845                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4846                                         "destination offset is too big"
4847                                         " or not aligned to 4 bytes");
4848                 if (action_modify_field->dst.level &&
4849                     action_modify_field->dst.field != RTE_FLOW_FIELD_TAG)
4850                         return rte_flow_error_set(error, ENOTSUP,
4851                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4852                                         "inner header fields modification"
4853                                         " is not supported");
4854         }
4855         if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE &&
4856             action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) {
4857                 if (!attr->transfer && !attr->group)
4858                         return rte_flow_error_set(error, ENOTSUP,
4859                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4860                                         "modify field action is not"
4861                                         " supported for group 0");
4862                 if ((action_modify_field->src.offset +
4863                      action_modify_field->width > src_width) ||
4864                     (action_modify_field->src.offset % 32))
4865                         return rte_flow_error_set(error, EINVAL,
4866                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4867                                         "source offset is too big"
4868                                         " or not aligned to 4 bytes");
4869                 if (action_modify_field->src.level &&
4870                     action_modify_field->src.field != RTE_FLOW_FIELD_TAG)
4871                         return rte_flow_error_set(error, ENOTSUP,
4872                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4873                                         "inner header fields modification"
4874                                         " is not supported");
4875         }
4876         if ((action_modify_field->dst.field ==
4877              action_modify_field->src.field) &&
4878             (action_modify_field->dst.level ==
4879              action_modify_field->src.level))
4880                 return rte_flow_error_set(error, EINVAL,
4881                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4882                                 "source and destination fields"
4883                                 " cannot be the same");
4884         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE ||
4885             action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER)
4886                 return rte_flow_error_set(error, EINVAL,
4887                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4888                                 "immediate value or a pointer to it"
4889                                 " cannot be used as a destination");
4890         if (action_modify_field->dst.field == RTE_FLOW_FIELD_START ||
4891             action_modify_field->src.field == RTE_FLOW_FIELD_START)
4892                 return rte_flow_error_set(error, ENOTSUP,
4893                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4894                                 "modifications of an arbitrary"
4895                                 " place in a packet is not supported");
4896         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE ||
4897             action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE)
4898                 return rte_flow_error_set(error, ENOTSUP,
4899                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4900                                 "modifications of the 802.1Q Tag"
4901                                 " Identifier is not supported");
4902         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI ||
4903             action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI)
4904                 return rte_flow_error_set(error, ENOTSUP,
4905                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4906                                 "modifications of the VXLAN Network"
4907                                 " Identifier is not supported");
4908         if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI ||
4909             action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI)
4910                 return rte_flow_error_set(error, ENOTSUP,
4911                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4912                                 "modifications of the GENEVE Network"
4913                                 " Identifier is not supported");
4914         if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK ||
4915             action_modify_field->src.field == RTE_FLOW_FIELD_MARK ||
4916             action_modify_field->dst.field == RTE_FLOW_FIELD_META ||
4917             action_modify_field->src.field == RTE_FLOW_FIELD_META) {
4918                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4919                     !mlx5_flow_ext_mreg_supported(dev))
4920                         return rte_flow_error_set(error, ENOTSUP,
4921                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4922                                         "cannot modify mark or metadata without"
4923                                         " extended metadata register support");
4924         }
4925         if (action_modify_field->operation != RTE_FLOW_MODIFY_SET)
4926                 return rte_flow_error_set(error, ENOTSUP,
4927                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4928                                 "add and sub operations"
4929                                 " are not supported");
4930         return (action_modify_field->width / 32) +
4931                !!(action_modify_field->width % 32);
4932 }
4933
4934 /**
4935  * Validate jump action.
4936  *
4937  * @param[in] action
4938  *   Pointer to the jump action.
4939  * @param[in] action_flags
4940  *   Holds the actions detected until now.
4941  * @param[in] attributes
4942  *   Pointer to flow attributes
4943  * @param[in] external
4944  *   Action belongs to flow rule created by request external to PMD.
4945  * @param[out] error
4946  *   Pointer to error structure.
4947  *
4948  * @return
4949  *   0 on success, a negative errno value otherwise and rte_errno is set.
4950  */
4951 static int
4952 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
4953                              const struct mlx5_flow_tunnel *tunnel,
4954                              const struct rte_flow_action *action,
4955                              uint64_t action_flags,
4956                              const struct rte_flow_attr *attributes,
4957                              bool external, struct rte_flow_error *error)
4958 {
4959         uint32_t target_group, table;
4960         int ret = 0;
4961         struct flow_grp_info grp_info = {
4962                 .external = !!external,
4963                 .transfer = !!attributes->transfer,
4964                 .fdb_def_rule = 1,
4965                 .std_tbl_fix = 0
4966         };
4967         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4968                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4969                 return rte_flow_error_set(error, EINVAL,
4970                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4971                                           "can't have 2 fate actions in"
4972                                           " same flow");
4973         if (!action->conf)
4974                 return rte_flow_error_set(error, EINVAL,
4975                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4976                                           NULL, "action configuration not set");
4977         target_group =
4978                 ((const struct rte_flow_action_jump *)action->conf)->group;
4979         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
4980                                        &grp_info, error);
4981         if (ret)
4982                 return ret;
4983         if (attributes->group == target_group &&
4984             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
4985                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
4986                 return rte_flow_error_set(error, EINVAL,
4987                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4988                                           "target group must be other than"
4989                                           " the current flow group");
4990         return 0;
4991 }
4992
4993 /*
4994  * Validate the port_id action.
4995  *
4996  * @param[in] dev
4997  *   Pointer to rte_eth_dev structure.
4998  * @param[in] action_flags
4999  *   Bit-fields that holds the actions detected until now.
5000  * @param[in] action
5001  *   Port_id RTE action structure.
5002  * @param[in] attr
5003  *   Attributes of flow that includes this action.
5004  * @param[out] error
5005  *   Pointer to error structure.
5006  *
5007  * @return
5008  *   0 on success, a negative errno value otherwise and rte_errno is set.
5009  */
5010 static int
5011 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
5012                                 uint64_t action_flags,
5013                                 const struct rte_flow_action *action,
5014                                 const struct rte_flow_attr *attr,
5015                                 struct rte_flow_error *error)
5016 {
5017         const struct rte_flow_action_port_id *port_id;
5018         struct mlx5_priv *act_priv;
5019         struct mlx5_priv *dev_priv;
5020         uint16_t port;
5021
5022         if (!attr->transfer)
5023                 return rte_flow_error_set(error, ENOTSUP,
5024                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5025                                           NULL,
5026                                           "port id action is valid in transfer"
5027                                           " mode only");
5028         if (!action || !action->conf)
5029                 return rte_flow_error_set(error, ENOTSUP,
5030                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5031                                           NULL,
5032                                           "port id action parameters must be"
5033                                           " specified");
5034         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
5035                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5036                 return rte_flow_error_set(error, EINVAL,
5037                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5038                                           "can have only one fate actions in"
5039                                           " a flow");
5040         dev_priv = mlx5_dev_to_eswitch_info(dev);
5041         if (!dev_priv)
5042                 return rte_flow_error_set(error, rte_errno,
5043                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5044                                           NULL,
5045                                           "failed to obtain E-Switch info");
5046         port_id = action->conf;
5047         port = port_id->original ? dev->data->port_id : port_id->id;
5048         act_priv = mlx5_port_to_eswitch_info(port, false);
5049         if (!act_priv)
5050                 return rte_flow_error_set
5051                                 (error, rte_errno,
5052                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
5053                                  "failed to obtain E-Switch port id for port");
5054         if (act_priv->domain_id != dev_priv->domain_id)
5055                 return rte_flow_error_set
5056                                 (error, EINVAL,
5057                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5058                                  "port does not belong to"
5059                                  " E-Switch being configured");
5060         return 0;
5061 }
5062
5063 /**
5064  * Get the maximum number of modify header actions.
5065  *
5066  * @param dev
5067  *   Pointer to rte_eth_dev structure.
5068  * @param root
5069  *   Whether action is on root table.
5070  *
5071  * @return
5072  *   Max number of modify header actions device can support.
5073  */
5074 static inline unsigned int
5075 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
5076                               bool root)
5077 {
5078         /*
5079          * There's no way to directly query the max capacity from FW.
5080          * The maximal value on root table should be assumed to be supported.
5081          */
5082         if (!root)
5083                 return MLX5_MAX_MODIFY_NUM;
5084         else
5085                 return MLX5_ROOT_TBL_MODIFY_NUM;
5086 }
5087
5088 /**
5089  * Validate the meter action.
5090  *
5091  * @param[in] dev
5092  *   Pointer to rte_eth_dev structure.
5093  * @param[in] action_flags
5094  *   Bit-fields that holds the actions detected until now.
5095  * @param[in] action
5096  *   Pointer to the meter action.
5097  * @param[in] attr
5098  *   Attributes of flow that includes this action.
5099  * @param[in] port_id_item
5100  *   Pointer to item indicating port id.
5101  * @param[out] error
5102  *   Pointer to error structure.
5103  *
5104  * @return
5105  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5106  */
5107 static int
5108 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
5109                                 uint64_t action_flags,
5110                                 const struct rte_flow_action *action,
5111                                 const struct rte_flow_attr *attr,
5112                                 const struct rte_flow_item *port_id_item,
5113                                 bool *def_policy,
5114                                 struct rte_flow_error *error)
5115 {
5116         struct mlx5_priv *priv = dev->data->dev_private;
5117         const struct rte_flow_action_meter *am = action->conf;
5118         struct mlx5_flow_meter_info *fm;
5119         struct mlx5_flow_meter_policy *mtr_policy;
5120         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
5121
5122         if (!am)
5123                 return rte_flow_error_set(error, EINVAL,
5124                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5125                                           "meter action conf is NULL");
5126
5127         if (action_flags & MLX5_FLOW_ACTION_METER)
5128                 return rte_flow_error_set(error, ENOTSUP,
5129                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5130                                           "meter chaining not support");
5131         if (action_flags & MLX5_FLOW_ACTION_JUMP)
5132                 return rte_flow_error_set(error, ENOTSUP,
5133                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5134                                           "meter with jump not support");
5135         if (!priv->mtr_en)
5136                 return rte_flow_error_set(error, ENOTSUP,
5137                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5138                                           NULL,
5139                                           "meter action not supported");
5140         fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL);
5141         if (!fm)
5142                 return rte_flow_error_set(error, EINVAL,
5143                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5144                                           "Meter not found");
5145         /* aso meter can always be shared by different domains */
5146         if (fm->ref_cnt && !priv->sh->meter_aso_en &&
5147             !(fm->transfer == attr->transfer ||
5148               (!fm->ingress && !attr->ingress && attr->egress) ||
5149               (!fm->egress && !attr->egress && attr->ingress)))
5150                 return rte_flow_error_set(error, EINVAL,
5151                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5152                         "Flow attributes domain are either invalid "
5153                         "or have a domain conflict with current "
5154                         "meter attributes");
5155         if (fm->def_policy) {
5156                 if (!((attr->transfer &&
5157                         mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) ||
5158                         (attr->egress &&
5159                         mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) ||
5160                         (attr->ingress &&
5161                         mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS])))
5162                         return rte_flow_error_set(error, EINVAL,
5163                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5164                                           "Flow attributes domain "
5165                                           "have a conflict with current "
5166                                           "meter domain attributes");
5167                 *def_policy = true;
5168         } else {
5169                 mtr_policy = mlx5_flow_meter_policy_find(dev,
5170                                                 fm->policy_id, NULL);
5171                 if (!mtr_policy)
5172                         return rte_flow_error_set(error, EINVAL,
5173                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5174                                           "Invalid policy id for meter ");
5175                 if (!((attr->transfer && mtr_policy->transfer) ||
5176                         (attr->egress && mtr_policy->egress) ||
5177                         (attr->ingress && mtr_policy->ingress)))
5178                         return rte_flow_error_set(error, EINVAL,
5179                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5180                                           "Flow attributes domain "
5181                                           "have a conflict with current "
5182                                           "meter domain attributes");
5183                 if (attr->transfer && mtr_policy->dev) {
5184                         /**
5185                          * When policy has fate action of port_id,
5186                          * the flow should have the same src port as policy.
5187                          */
5188                         struct mlx5_priv *policy_port_priv =
5189                                         mtr_policy->dev->data->dev_private;
5190                         int32_t flow_src_port = priv->representor_id;
5191
5192                         if (port_id_item) {
5193                                 const struct rte_flow_item_port_id *spec =
5194                                                         port_id_item->spec;
5195                                 struct mlx5_priv *port_priv =
5196                                         mlx5_port_to_eswitch_info(spec->id,
5197                                                                   false);
5198                                 if (!port_priv)
5199                                         return rte_flow_error_set(error,
5200                                                 rte_errno,
5201                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5202                                                 spec,
5203                                                 "Failed to get port info.");
5204                                 flow_src_port = port_priv->representor_id;
5205                         }
5206                         if (flow_src_port != policy_port_priv->representor_id)
5207                                 return rte_flow_error_set(error,
5208                                                 rte_errno,
5209                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5210                                                 NULL,
5211                                                 "Flow and meter policy "
5212                                                 "have different src port.");
5213                 }
5214                 *def_policy = false;
5215         }
5216         return 0;
5217 }
5218
5219 /**
5220  * Validate the age action.
5221  *
5222  * @param[in] action_flags
5223  *   Holds the actions detected until now.
5224  * @param[in] action
5225  *   Pointer to the age action.
5226  * @param[in] dev
5227  *   Pointer to the Ethernet device structure.
5228  * @param[out] error
5229  *   Pointer to error structure.
5230  *
5231  * @return
5232  *   0 on success, a negative errno value otherwise and rte_errno is set.
5233  */
5234 static int
5235 flow_dv_validate_action_age(uint64_t action_flags,
5236                             const struct rte_flow_action *action,
5237                             struct rte_eth_dev *dev,
5238                             struct rte_flow_error *error)
5239 {
5240         struct mlx5_priv *priv = dev->data->dev_private;
5241         const struct rte_flow_action_age *age = action->conf;
5242
5243         if (!priv->config.devx || (priv->sh->cmng.counter_fallback &&
5244             !priv->sh->aso_age_mng))
5245                 return rte_flow_error_set(error, ENOTSUP,
5246                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5247                                           NULL,
5248                                           "age action not supported");
5249         if (!(action->conf))
5250                 return rte_flow_error_set(error, EINVAL,
5251                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5252                                           "configuration cannot be null");
5253         if (!(age->timeout))
5254                 return rte_flow_error_set(error, EINVAL,
5255                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5256                                           "invalid timeout value 0");
5257         if (action_flags & MLX5_FLOW_ACTION_AGE)
5258                 return rte_flow_error_set(error, EINVAL,
5259                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5260                                           "duplicate age actions set");
5261         return 0;
5262 }
5263
5264 /**
5265  * Validate the modify-header IPv4 DSCP actions.
5266  *
5267  * @param[in] action_flags
5268  *   Holds the actions detected until now.
5269  * @param[in] action
5270  *   Pointer to the modify action.
5271  * @param[in] item_flags
5272  *   Holds the items detected.
5273  * @param[out] error
5274  *   Pointer to error structure.
5275  *
5276  * @return
5277  *   0 on success, a negative errno value otherwise and rte_errno is set.
5278  */
5279 static int
5280 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
5281                                          const struct rte_flow_action *action,
5282                                          const uint64_t item_flags,
5283                                          struct rte_flow_error *error)
5284 {
5285         int ret = 0;
5286
5287         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5288         if (!ret) {
5289                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
5290                         return rte_flow_error_set(error, EINVAL,
5291                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5292                                                   NULL,
5293                                                   "no ipv4 item in pattern");
5294         }
5295         return ret;
5296 }
5297
5298 /**
5299  * Validate the modify-header IPv6 DSCP actions.
5300  *
5301  * @param[in] action_flags
5302  *   Holds the actions detected until now.
5303  * @param[in] action
5304  *   Pointer to the modify action.
5305  * @param[in] item_flags
5306  *   Holds the items detected.
5307  * @param[out] error
5308  *   Pointer to error structure.
5309  *
5310  * @return
5311  *   0 on success, a negative errno value otherwise and rte_errno is set.
5312  */
5313 static int
5314 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
5315                                          const struct rte_flow_action *action,
5316                                          const uint64_t item_flags,
5317                                          struct rte_flow_error *error)
5318 {
5319         int ret = 0;
5320
5321         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5322         if (!ret) {
5323                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
5324                         return rte_flow_error_set(error, EINVAL,
5325                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5326                                                   NULL,
5327                                                   "no ipv6 item in pattern");
5328         }
5329         return ret;
5330 }
5331
5332 int
5333 flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
5334                         struct mlx5_list_entry *entry, void *cb_ctx)
5335 {
5336         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5337         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5338         struct mlx5_flow_dv_modify_hdr_resource *resource =
5339                                   container_of(entry, typeof(*resource), entry);
5340         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5341
5342         key_len += ref->actions_num * sizeof(ref->actions[0]);
5343         return ref->actions_num != resource->actions_num ||
5344                memcmp(&ref->ft_type, &resource->ft_type, key_len);
5345 }
5346
5347 struct mlx5_list_entry *
5348 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
5349 {
5350         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5351         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5352         struct mlx5dv_dr_domain *ns;
5353         struct mlx5_flow_dv_modify_hdr_resource *entry;
5354         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5355         int ret;
5356         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5357         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5358
5359         entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0,
5360                             SOCKET_ID_ANY);
5361         if (!entry) {
5362                 rte_flow_error_set(ctx->error, ENOMEM,
5363                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5364                                    "cannot allocate resource memory");
5365                 return NULL;
5366         }
5367         rte_memcpy(&entry->ft_type,
5368                    RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
5369                    key_len + data_len);
5370         if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
5371                 ns = sh->fdb_domain;
5372         else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
5373                 ns = sh->tx_domain;
5374         else
5375                 ns = sh->rx_domain;
5376         ret = mlx5_flow_os_create_flow_action_modify_header
5377                                         (sh->ctx, ns, entry,
5378                                          data_len, &entry->action);
5379         if (ret) {
5380                 mlx5_free(entry);
5381                 rte_flow_error_set(ctx->error, ENOMEM,
5382                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5383                                    NULL, "cannot create modification action");
5384                 return NULL;
5385         }
5386         return &entry->entry;
5387 }
5388
5389 struct mlx5_list_entry *
5390 flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
5391                         struct mlx5_list_entry *oentry, void *cb_ctx)
5392 {
5393         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5394         struct mlx5_flow_dv_modify_hdr_resource *entry;
5395         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5396         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5397
5398         entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
5399         if (!entry) {
5400                 rte_flow_error_set(ctx->error, ENOMEM,
5401                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5402                                    "cannot allocate resource memory");
5403                 return NULL;
5404         }
5405         memcpy(entry, oentry, sizeof(*entry) + data_len);
5406         return &entry->entry;
5407 }
5408
5409 void
5410 flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
5411                              struct mlx5_list_entry *entry)
5412 {
5413         mlx5_free(entry);
5414 }
5415
5416 /**
5417  * Validate the sample action.
5418  *
5419  * @param[in, out] action_flags
5420  *   Holds the actions detected until now.
5421  * @param[in] action
5422  *   Pointer to the sample action.
5423  * @param[in] dev
5424  *   Pointer to the Ethernet device structure.
5425  * @param[in] attr
5426  *   Attributes of flow that includes this action.
5427  * @param[in] item_flags
5428  *   Holds the items detected.
5429  * @param[in] rss
5430  *   Pointer to the RSS action.
5431  * @param[out] sample_rss
5432  *   Pointer to the RSS action in sample action list.
5433  * @param[out] count
5434  *   Pointer to the COUNT action in sample action list.
5435  * @param[out] fdb_mirror_limit
5436  *   Pointer to the FDB mirror limitation flag.
5437  * @param[out] error
5438  *   Pointer to error structure.
5439  *
5440  * @return
5441  *   0 on success, a negative errno value otherwise and rte_errno is set.
5442  */
5443 static int
5444 flow_dv_validate_action_sample(uint64_t *action_flags,
5445                                const struct rte_flow_action *action,
5446                                struct rte_eth_dev *dev,
5447                                const struct rte_flow_attr *attr,
5448                                uint64_t item_flags,
5449                                const struct rte_flow_action_rss *rss,
5450                                const struct rte_flow_action_rss **sample_rss,
5451                                const struct rte_flow_action_count **count,
5452                                int *fdb_mirror_limit,
5453                                struct rte_flow_error *error)
5454 {
5455         struct mlx5_priv *priv = dev->data->dev_private;
5456         struct mlx5_dev_config *dev_conf = &priv->config;
5457         const struct rte_flow_action_sample *sample = action->conf;
5458         const struct rte_flow_action *act;
5459         uint64_t sub_action_flags = 0;
5460         uint16_t queue_index = 0xFFFF;
5461         int actions_n = 0;
5462         int ret;
5463
5464         if (!sample)
5465                 return rte_flow_error_set(error, EINVAL,
5466                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5467                                           "configuration cannot be NULL");
5468         if (sample->ratio == 0)
5469                 return rte_flow_error_set(error, EINVAL,
5470                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5471                                           "ratio value starts from 1");
5472         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
5473                 return rte_flow_error_set(error, ENOTSUP,
5474                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5475                                           NULL,
5476                                           "sample action not supported");
5477         if (*action_flags & MLX5_FLOW_ACTION_SAMPLE)
5478                 return rte_flow_error_set(error, EINVAL,
5479                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5480                                           "Multiple sample actions not "
5481                                           "supported");
5482         if (*action_flags & MLX5_FLOW_ACTION_METER)
5483                 return rte_flow_error_set(error, EINVAL,
5484                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5485                                           "wrong action order, meter should "
5486                                           "be after sample action");
5487         if (*action_flags & MLX5_FLOW_ACTION_JUMP)
5488                 return rte_flow_error_set(error, EINVAL,
5489                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5490                                           "wrong action order, jump should "
5491                                           "be after sample action");
5492         act = sample->actions;
5493         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
5494                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5495                         return rte_flow_error_set(error, ENOTSUP,
5496                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5497                                                   act, "too many actions");
5498                 switch (act->type) {
5499                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5500                         ret = mlx5_flow_validate_action_queue(act,
5501                                                               sub_action_flags,
5502                                                               dev,
5503                                                               attr, error);
5504                         if (ret < 0)
5505                                 return ret;
5506                         queue_index = ((const struct rte_flow_action_queue *)
5507                                                         (act->conf))->index;
5508                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
5509                         ++actions_n;
5510                         break;
5511                 case RTE_FLOW_ACTION_TYPE_RSS:
5512                         *sample_rss = act->conf;
5513                         ret = mlx5_flow_validate_action_rss(act,
5514                                                             sub_action_flags,
5515                                                             dev, attr,
5516                                                             item_flags,
5517                                                             error);
5518                         if (ret < 0)
5519                                 return ret;
5520                         if (rss && *sample_rss &&
5521                             ((*sample_rss)->level != rss->level ||
5522                             (*sample_rss)->types != rss->types))
5523                                 return rte_flow_error_set(error, ENOTSUP,
5524                                         RTE_FLOW_ERROR_TYPE_ACTION,
5525                                         NULL,
5526                                         "Can't use the different RSS types "
5527                                         "or level in the same flow");
5528                         if (*sample_rss != NULL && (*sample_rss)->queue_num)
5529                                 queue_index = (*sample_rss)->queue[0];
5530                         sub_action_flags |= MLX5_FLOW_ACTION_RSS;
5531                         ++actions_n;
5532                         break;
5533                 case RTE_FLOW_ACTION_TYPE_MARK:
5534                         ret = flow_dv_validate_action_mark(dev, act,
5535                                                            sub_action_flags,
5536                                                            attr, error);
5537                         if (ret < 0)
5538                                 return ret;
5539                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
5540                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
5541                                                 MLX5_FLOW_ACTION_MARK_EXT;
5542                         else
5543                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
5544                         ++actions_n;
5545                         break;
5546                 case RTE_FLOW_ACTION_TYPE_COUNT:
5547                         ret = flow_dv_validate_action_count
5548                                 (dev, is_shared_action_count(act),
5549                                  *action_flags | sub_action_flags,
5550                                  error);
5551                         if (ret < 0)
5552                                 return ret;
5553                         *count = act->conf;
5554                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
5555                         *action_flags |= MLX5_FLOW_ACTION_COUNT;
5556                         ++actions_n;
5557                         break;
5558                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5559                         ret = flow_dv_validate_action_port_id(dev,
5560                                                               sub_action_flags,
5561                                                               act,
5562                                                               attr,
5563                                                               error);
5564                         if (ret)
5565                                 return ret;
5566                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5567                         ++actions_n;
5568                         break;
5569                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5570                         ret = flow_dv_validate_action_raw_encap_decap
5571                                 (dev, NULL, act->conf, attr, &sub_action_flags,
5572                                  &actions_n, action, item_flags, error);
5573                         if (ret < 0)
5574                                 return ret;
5575                         ++actions_n;
5576                         break;
5577                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5578                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5579                         ret = flow_dv_validate_action_l2_encap(dev,
5580                                                                sub_action_flags,
5581                                                                act, attr,
5582                                                                error);
5583                         if (ret < 0)
5584                                 return ret;
5585                         sub_action_flags |= MLX5_FLOW_ACTION_ENCAP;
5586                         ++actions_n;
5587                         break;
5588                 default:
5589                         return rte_flow_error_set(error, ENOTSUP,
5590                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5591                                                   NULL,
5592                                                   "Doesn't support optional "
5593                                                   "action");
5594                 }
5595         }
5596         if (attr->ingress && !attr->transfer) {
5597                 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE |
5598                                           MLX5_FLOW_ACTION_RSS)))
5599                         return rte_flow_error_set(error, EINVAL,
5600                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5601                                                   NULL,
5602                                                   "Ingress must has a dest "
5603                                                   "QUEUE for Sample");
5604         } else if (attr->egress && !attr->transfer) {
5605                 return rte_flow_error_set(error, ENOTSUP,
5606                                           RTE_FLOW_ERROR_TYPE_ACTION,
5607                                           NULL,
5608                                           "Sample Only support Ingress "
5609                                           "or E-Switch");
5610         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
5611                 MLX5_ASSERT(attr->transfer);
5612                 if (sample->ratio > 1)
5613                         return rte_flow_error_set(error, ENOTSUP,
5614                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5615                                                   NULL,
5616                                                   "E-Switch doesn't support "
5617                                                   "any optional action "
5618                                                   "for sampling");
5619                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
5620                         return rte_flow_error_set(error, ENOTSUP,
5621                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5622                                                   NULL,
5623                                                   "unsupported action QUEUE");
5624                 if (sub_action_flags & MLX5_FLOW_ACTION_RSS)
5625                         return rte_flow_error_set(error, ENOTSUP,
5626                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5627                                                   NULL,
5628                                                   "unsupported action QUEUE");
5629                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
5630                         return rte_flow_error_set(error, EINVAL,
5631                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5632                                                   NULL,
5633                                                   "E-Switch must has a dest "
5634                                                   "port for mirroring");
5635                 if (!priv->config.hca_attr.reg_c_preserve &&
5636                      priv->representor_id != UINT16_MAX)
5637                         *fdb_mirror_limit = 1;
5638         }
5639         /* Continue validation for Xcap actions.*/
5640         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
5641             (queue_index == 0xFFFF ||
5642              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5643                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5644                      MLX5_FLOW_XCAP_ACTIONS)
5645                         return rte_flow_error_set(error, ENOTSUP,
5646                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5647                                                   NULL, "encap and decap "
5648                                                   "combination aren't "
5649                                                   "supported");
5650                 if (!attr->transfer && attr->ingress && (sub_action_flags &
5651                                                         MLX5_FLOW_ACTION_ENCAP))
5652                         return rte_flow_error_set(error, ENOTSUP,
5653                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5654                                                   NULL, "encap is not supported"
5655                                                   " for ingress traffic");
5656         }
5657         return 0;
5658 }
5659
5660 /**
5661  * Find existing modify-header resource or create and register a new one.
5662  *
5663  * @param dev[in, out]
5664  *   Pointer to rte_eth_dev structure.
5665  * @param[in, out] resource
5666  *   Pointer to modify-header resource.
5667  * @parm[in, out] dev_flow
5668  *   Pointer to the dev_flow.
5669  * @param[out] error
5670  *   pointer to error structure.
5671  *
5672  * @return
5673  *   0 on success otherwise -errno and errno is set.
5674  */
5675 static int
5676 flow_dv_modify_hdr_resource_register
5677                         (struct rte_eth_dev *dev,
5678                          struct mlx5_flow_dv_modify_hdr_resource *resource,
5679                          struct mlx5_flow *dev_flow,
5680                          struct rte_flow_error *error)
5681 {
5682         struct mlx5_priv *priv = dev->data->dev_private;
5683         struct mlx5_dev_ctx_shared *sh = priv->sh;
5684         uint32_t key_len = sizeof(*resource) -
5685                            offsetof(typeof(*resource), ft_type) +
5686                            resource->actions_num * sizeof(resource->actions[0]);
5687         struct mlx5_list_entry *entry;
5688         struct mlx5_flow_cb_ctx ctx = {
5689                 .error = error,
5690                 .data = resource,
5691         };
5692         uint64_t key64;
5693
5694         resource->root = !dev_flow->dv.group;
5695         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
5696                                                                 resource->root))
5697                 return rte_flow_error_set(error, EOVERFLOW,
5698                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5699                                           "too many modify header items");
5700         key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
5701         entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
5702         if (!entry)
5703                 return -rte_errno;
5704         resource = container_of(entry, typeof(*resource), entry);
5705         dev_flow->handle->dvh.modify_hdr = resource;
5706         return 0;
5707 }
5708
5709 /**
5710  * Get DV flow counter by index.
5711  *
5712  * @param[in] dev
5713  *   Pointer to the Ethernet device structure.
5714  * @param[in] idx
5715  *   mlx5 flow counter index in the container.
5716  * @param[out] ppool
5717  *   mlx5 flow counter pool in the container.
5718  *
5719  * @return
5720  *   Pointer to the counter, NULL otherwise.
5721  */
5722 static struct mlx5_flow_counter *
5723 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
5724                            uint32_t idx,
5725                            struct mlx5_flow_counter_pool **ppool)
5726 {
5727         struct mlx5_priv *priv = dev->data->dev_private;
5728         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5729         struct mlx5_flow_counter_pool *pool;
5730
5731         /* Decrease to original index and clear shared bit. */
5732         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
5733         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
5734         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
5735         MLX5_ASSERT(pool);
5736         if (ppool)
5737                 *ppool = pool;
5738         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
5739 }
5740
5741 /**
5742  * Check the devx counter belongs to the pool.
5743  *
5744  * @param[in] pool
5745  *   Pointer to the counter pool.
5746  * @param[in] id
5747  *   The counter devx ID.
5748  *
5749  * @return
5750  *   True if counter belongs to the pool, false otherwise.
5751  */
5752 static bool
5753 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
5754 {
5755         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
5756                    MLX5_COUNTERS_PER_POOL;
5757
5758         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
5759                 return true;
5760         return false;
5761 }
5762
5763 /**
5764  * Get a pool by devx counter ID.
5765  *
5766  * @param[in] cmng
5767  *   Pointer to the counter management.
5768  * @param[in] id
5769  *   The counter devx ID.
5770  *
5771  * @return
5772  *   The counter pool pointer if exists, NULL otherwise,
5773  */
5774 static struct mlx5_flow_counter_pool *
5775 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
5776 {
5777         uint32_t i;
5778         struct mlx5_flow_counter_pool *pool = NULL;
5779
5780         rte_spinlock_lock(&cmng->pool_update_sl);
5781         /* Check last used pool. */
5782         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
5783             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
5784                 pool = cmng->pools[cmng->last_pool_idx];
5785                 goto out;
5786         }
5787         /* ID out of range means no suitable pool in the container. */
5788         if (id > cmng->max_id || id < cmng->min_id)
5789                 goto out;
5790         /*
5791          * Find the pool from the end of the container, since mostly counter
5792          * ID is sequence increasing, and the last pool should be the needed
5793          * one.
5794          */
5795         i = cmng->n_valid;
5796         while (i--) {
5797                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
5798
5799                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
5800                         pool = pool_tmp;
5801                         break;
5802                 }
5803         }
5804 out:
5805         rte_spinlock_unlock(&cmng->pool_update_sl);
5806         return pool;
5807 }
5808
5809 /**
5810  * Resize a counter container.
5811  *
5812  * @param[in] dev
5813  *   Pointer to the Ethernet device structure.
5814  *
5815  * @return
5816  *   0 on success, otherwise negative errno value and rte_errno is set.
5817  */
5818 static int
5819 flow_dv_container_resize(struct rte_eth_dev *dev)
5820 {
5821         struct mlx5_priv *priv = dev->data->dev_private;
5822         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5823         void *old_pools = cmng->pools;
5824         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
5825         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
5826         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
5827
5828         if (!pools) {
5829                 rte_errno = ENOMEM;
5830                 return -ENOMEM;
5831         }
5832         if (old_pools)
5833                 memcpy(pools, old_pools, cmng->n *
5834                                        sizeof(struct mlx5_flow_counter_pool *));
5835         cmng->n = resize;
5836         cmng->pools = pools;
5837         if (old_pools)
5838                 mlx5_free(old_pools);
5839         return 0;
5840 }
5841
5842 /**
5843  * Query a devx flow counter.
5844  *
5845  * @param[in] dev
5846  *   Pointer to the Ethernet device structure.
5847  * @param[in] counter
5848  *   Index to the flow counter.
5849  * @param[out] pkts
5850  *   The statistics value of packets.
5851  * @param[out] bytes
5852  *   The statistics value of bytes.
5853  *
5854  * @return
5855  *   0 on success, otherwise a negative errno value and rte_errno is set.
5856  */
5857 static inline int
5858 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
5859                      uint64_t *bytes)
5860 {
5861         struct mlx5_priv *priv = dev->data->dev_private;
5862         struct mlx5_flow_counter_pool *pool = NULL;
5863         struct mlx5_flow_counter *cnt;
5864         int offset;
5865
5866         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5867         MLX5_ASSERT(pool);
5868         if (priv->sh->cmng.counter_fallback)
5869                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
5870                                         0, pkts, bytes, 0, NULL, NULL, 0);
5871         rte_spinlock_lock(&pool->sl);
5872         if (!pool->raw) {
5873                 *pkts = 0;
5874                 *bytes = 0;
5875         } else {
5876                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
5877                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
5878                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
5879         }
5880         rte_spinlock_unlock(&pool->sl);
5881         return 0;
5882 }
5883
5884 /**
5885  * Create and initialize a new counter pool.
5886  *
5887  * @param[in] dev
5888  *   Pointer to the Ethernet device structure.
5889  * @param[out] dcs
5890  *   The devX counter handle.
5891  * @param[in] age
5892  *   Whether the pool is for counter that was allocated for aging.
5893  * @param[in/out] cont_cur
5894  *   Pointer to the container pointer, it will be update in pool resize.
5895  *
5896  * @return
5897  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
5898  */
5899 static struct mlx5_flow_counter_pool *
5900 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
5901                     uint32_t age)
5902 {
5903         struct mlx5_priv *priv = dev->data->dev_private;
5904         struct mlx5_flow_counter_pool *pool;
5905         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5906         bool fallback = priv->sh->cmng.counter_fallback;
5907         uint32_t size = sizeof(*pool);
5908
5909         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
5910         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
5911         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
5912         if (!pool) {
5913                 rte_errno = ENOMEM;
5914                 return NULL;
5915         }
5916         pool->raw = NULL;
5917         pool->is_aged = !!age;
5918         pool->query_gen = 0;
5919         pool->min_dcs = dcs;
5920         rte_spinlock_init(&pool->sl);
5921         rte_spinlock_init(&pool->csl);
5922         TAILQ_INIT(&pool->counters[0]);
5923         TAILQ_INIT(&pool->counters[1]);
5924         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
5925         rte_spinlock_lock(&cmng->pool_update_sl);
5926         pool->index = cmng->n_valid;
5927         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
5928                 mlx5_free(pool);
5929                 rte_spinlock_unlock(&cmng->pool_update_sl);
5930                 return NULL;
5931         }
5932         cmng->pools[pool->index] = pool;
5933         cmng->n_valid++;
5934         if (unlikely(fallback)) {
5935                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
5936
5937                 if (base < cmng->min_id)
5938                         cmng->min_id = base;
5939                 if (base > cmng->max_id)
5940                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
5941                 cmng->last_pool_idx = pool->index;
5942         }
5943         rte_spinlock_unlock(&cmng->pool_update_sl);
5944         return pool;
5945 }
5946
5947 /**
5948  * Prepare a new counter and/or a new counter pool.
5949  *
5950  * @param[in] dev
5951  *   Pointer to the Ethernet device structure.
5952  * @param[out] cnt_free
5953  *   Where to put the pointer of a new counter.
5954  * @param[in] age
5955  *   Whether the pool is for counter that was allocated for aging.
5956  *
5957  * @return
5958  *   The counter pool pointer and @p cnt_free is set on success,
5959  *   NULL otherwise and rte_errno is set.
5960  */
5961 static struct mlx5_flow_counter_pool *
5962 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
5963                              struct mlx5_flow_counter **cnt_free,
5964                              uint32_t age)
5965 {
5966         struct mlx5_priv *priv = dev->data->dev_private;
5967         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5968         struct mlx5_flow_counter_pool *pool;
5969         struct mlx5_counters tmp_tq;
5970         struct mlx5_devx_obj *dcs = NULL;
5971         struct mlx5_flow_counter *cnt;
5972         enum mlx5_counter_type cnt_type =
5973                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
5974         bool fallback = priv->sh->cmng.counter_fallback;
5975         uint32_t i;
5976
5977         if (fallback) {
5978                 /* bulk_bitmap must be 0 for single counter allocation. */
5979                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
5980                 if (!dcs)
5981                         return NULL;
5982                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
5983                 if (!pool) {
5984                         pool = flow_dv_pool_create(dev, dcs, age);
5985                         if (!pool) {
5986                                 mlx5_devx_cmd_destroy(dcs);
5987                                 return NULL;
5988                         }
5989                 }
5990                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
5991                 cnt = MLX5_POOL_GET_CNT(pool, i);
5992                 cnt->pool = pool;
5993                 cnt->dcs_when_free = dcs;
5994                 *cnt_free = cnt;
5995                 return pool;
5996         }
5997         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
5998         if (!dcs) {
5999                 rte_errno = ENODATA;
6000                 return NULL;
6001         }
6002         pool = flow_dv_pool_create(dev, dcs, age);
6003         if (!pool) {
6004                 mlx5_devx_cmd_destroy(dcs);
6005                 return NULL;
6006         }
6007         TAILQ_INIT(&tmp_tq);
6008         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
6009                 cnt = MLX5_POOL_GET_CNT(pool, i);
6010                 cnt->pool = pool;
6011                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
6012         }
6013         rte_spinlock_lock(&cmng->csl[cnt_type]);
6014         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
6015         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6016         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
6017         (*cnt_free)->pool = pool;
6018         return pool;
6019 }
6020
6021 /**
6022  * Allocate a flow counter.
6023  *
6024  * @param[in] dev
6025  *   Pointer to the Ethernet device structure.
6026  * @param[in] age
6027  *   Whether the counter was allocated for aging.
6028  *
6029  * @return
6030  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6031  */
6032 static uint32_t
6033 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
6034 {
6035         struct mlx5_priv *priv = dev->data->dev_private;
6036         struct mlx5_flow_counter_pool *pool = NULL;
6037         struct mlx5_flow_counter *cnt_free = NULL;
6038         bool fallback = priv->sh->cmng.counter_fallback;
6039         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6040         enum mlx5_counter_type cnt_type =
6041                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6042         uint32_t cnt_idx;
6043
6044         if (!priv->config.devx) {
6045                 rte_errno = ENOTSUP;
6046                 return 0;
6047         }
6048         /* Get free counters from container. */
6049         rte_spinlock_lock(&cmng->csl[cnt_type]);
6050         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
6051         if (cnt_free)
6052                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
6053         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6054         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
6055                 goto err;
6056         pool = cnt_free->pool;
6057         if (fallback)
6058                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
6059         /* Create a DV counter action only in the first time usage. */
6060         if (!cnt_free->action) {
6061                 uint16_t offset;
6062                 struct mlx5_devx_obj *dcs;
6063                 int ret;
6064
6065                 if (!fallback) {
6066                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
6067                         dcs = pool->min_dcs;
6068                 } else {
6069                         offset = 0;
6070                         dcs = cnt_free->dcs_when_free;
6071                 }
6072                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
6073                                                             &cnt_free->action);
6074                 if (ret) {
6075                         rte_errno = errno;
6076                         goto err;
6077                 }
6078         }
6079         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
6080                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
6081         /* Update the counter reset values. */
6082         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
6083                                  &cnt_free->bytes))
6084                 goto err;
6085         if (!fallback && !priv->sh->cmng.query_thread_on)
6086                 /* Start the asynchronous batch query by the host thread. */
6087                 mlx5_set_query_alarm(priv->sh);
6088         /*
6089          * When the count action isn't shared (by ID), shared_info field is
6090          * used for indirect action API's refcnt.
6091          * When the counter action is not shared neither by ID nor by indirect
6092          * action API, shared info must be 1.
6093          */
6094         cnt_free->shared_info.refcnt = 1;
6095         return cnt_idx;
6096 err:
6097         if (cnt_free) {
6098                 cnt_free->pool = pool;
6099                 if (fallback)
6100                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
6101                 rte_spinlock_lock(&cmng->csl[cnt_type]);
6102                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
6103                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
6104         }
6105         return 0;
6106 }
6107
6108 /**
6109  * Allocate a shared flow counter.
6110  *
6111  * @param[in] ctx
6112  *   Pointer to the shared counter configuration.
6113  * @param[in] data
6114  *   Pointer to save the allocated counter index.
6115  *
6116  * @return
6117  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6118  */
6119
6120 static int32_t
6121 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
6122 {
6123         struct mlx5_shared_counter_conf *conf = ctx;
6124         struct rte_eth_dev *dev = conf->dev;
6125         struct mlx5_flow_counter *cnt;
6126
6127         data->dword = flow_dv_counter_alloc(dev, 0);
6128         data->dword |= MLX5_CNT_SHARED_OFFSET;
6129         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
6130         cnt->shared_info.id = conf->id;
6131         return 0;
6132 }
6133
6134 /**
6135  * Get a shared flow counter.
6136  *
6137  * @param[in] dev
6138  *   Pointer to the Ethernet device structure.
6139  * @param[in] id
6140  *   Counter identifier.
6141  *
6142  * @return
6143  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6144  */
6145 static uint32_t
6146 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
6147 {
6148         struct mlx5_priv *priv = dev->data->dev_private;
6149         struct mlx5_shared_counter_conf conf = {
6150                 .dev = dev,
6151                 .id = id,
6152         };
6153         union mlx5_l3t_data data = {
6154                 .dword = 0,
6155         };
6156
6157         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
6158                                flow_dv_counter_alloc_shared_cb, &conf);
6159         return data.dword;
6160 }
6161
6162 /**
6163  * Get age param from counter index.
6164  *
6165  * @param[in] dev
6166  *   Pointer to the Ethernet device structure.
6167  * @param[in] counter
6168  *   Index to the counter handler.
6169  *
6170  * @return
6171  *   The aging parameter specified for the counter index.
6172  */
6173 static struct mlx5_age_param*
6174 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
6175                                 uint32_t counter)
6176 {
6177         struct mlx5_flow_counter *cnt;
6178         struct mlx5_flow_counter_pool *pool = NULL;
6179
6180         flow_dv_counter_get_by_idx(dev, counter, &pool);
6181         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
6182         cnt = MLX5_POOL_GET_CNT(pool, counter);
6183         return MLX5_CNT_TO_AGE(cnt);
6184 }
6185
6186 /**
6187  * Remove a flow counter from aged counter list.
6188  *
6189  * @param[in] dev
6190  *   Pointer to the Ethernet device structure.
6191  * @param[in] counter
6192  *   Index to the counter handler.
6193  * @param[in] cnt
6194  *   Pointer to the counter handler.
6195  */
6196 static void
6197 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
6198                                 uint32_t counter, struct mlx5_flow_counter *cnt)
6199 {
6200         struct mlx5_age_info *age_info;
6201         struct mlx5_age_param *age_param;
6202         struct mlx5_priv *priv = dev->data->dev_private;
6203         uint16_t expected = AGE_CANDIDATE;
6204
6205         age_info = GET_PORT_AGE_INFO(priv);
6206         age_param = flow_dv_counter_idx_get_age(dev, counter);
6207         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
6208                                          AGE_FREE, false, __ATOMIC_RELAXED,
6209                                          __ATOMIC_RELAXED)) {
6210                 /**
6211                  * We need the lock even it is age timeout,
6212                  * since counter may still in process.
6213                  */
6214                 rte_spinlock_lock(&age_info->aged_sl);
6215                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
6216                 rte_spinlock_unlock(&age_info->aged_sl);
6217                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
6218         }
6219 }
6220
6221 /**
6222  * Release a flow counter.
6223  *
6224  * @param[in] dev
6225  *   Pointer to the Ethernet device structure.
6226  * @param[in] counter
6227  *   Index to the counter handler.
6228  */
6229 static void
6230 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter)
6231 {
6232         struct mlx5_priv *priv = dev->data->dev_private;
6233         struct mlx5_flow_counter_pool *pool = NULL;
6234         struct mlx5_flow_counter *cnt;
6235         enum mlx5_counter_type cnt_type;
6236
6237         if (!counter)
6238                 return;
6239         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6240         MLX5_ASSERT(pool);
6241         if (pool->is_aged) {
6242                 flow_dv_counter_remove_from_age(dev, counter, cnt);
6243         } else {
6244                 /*
6245                  * If the counter action is shared by ID, the l3t_clear_entry
6246                  * function reduces its references counter. If after the
6247                  * reduction the action is still referenced, the function
6248                  * returns here and does not release it.
6249                  */
6250                 if (IS_LEGACY_SHARED_CNT(counter) &&
6251                     mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
6252                                          cnt->shared_info.id))
6253                         return;
6254                 /*
6255                  * If the counter action is shared by indirect action API,
6256                  * the atomic function reduces its references counter.
6257                  * If after the reduction the action is still referenced, the
6258                  * function returns here and does not release it.
6259                  * When the counter action is not shared neither by ID nor by
6260                  * indirect action API, shared info is 1 before the reduction,
6261                  * so this condition is failed and function doesn't return here.
6262                  */
6263                 if (!IS_LEGACY_SHARED_CNT(counter) &&
6264                     __atomic_sub_fetch(&cnt->shared_info.refcnt, 1,
6265                                        __ATOMIC_RELAXED))
6266                         return;
6267         }
6268         cnt->pool = pool;
6269         /*
6270          * Put the counter back to list to be updated in none fallback mode.
6271          * Currently, we are using two list alternately, while one is in query,
6272          * add the freed counter to the other list based on the pool query_gen
6273          * value. After query finishes, add counter the list to the global
6274          * container counter list. The list changes while query starts. In
6275          * this case, lock will not be needed as query callback and release
6276          * function both operate with the different list.
6277          */
6278         if (!priv->sh->cmng.counter_fallback) {
6279                 rte_spinlock_lock(&pool->csl);
6280                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
6281                 rte_spinlock_unlock(&pool->csl);
6282         } else {
6283                 cnt->dcs_when_free = cnt->dcs_when_active;
6284                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6285                                            MLX5_COUNTER_TYPE_ORIGIN;
6286                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
6287                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
6288                                   cnt, next);
6289                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
6290         }
6291 }
6292
6293 /**
6294  * Resize a meter id container.
6295  *
6296  * @param[in] dev
6297  *   Pointer to the Ethernet device structure.
6298  *
6299  * @return
6300  *   0 on success, otherwise negative errno value and rte_errno is set.
6301  */
6302 static int
6303 flow_dv_mtr_container_resize(struct rte_eth_dev *dev)
6304 {
6305         struct mlx5_priv *priv = dev->data->dev_private;
6306         struct mlx5_aso_mtr_pools_mng *pools_mng =
6307                                 &priv->sh->mtrmng->pools_mng;
6308         void *old_pools = pools_mng->pools;
6309         uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE;
6310         uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize;
6311         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
6312
6313         if (!pools) {
6314                 rte_errno = ENOMEM;
6315                 return -ENOMEM;
6316         }
6317         if (!pools_mng->n)
6318                 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) {
6319                         mlx5_free(pools);
6320                         return -ENOMEM;
6321                 }
6322         if (old_pools)
6323                 memcpy(pools, old_pools, pools_mng->n *
6324                                        sizeof(struct mlx5_aso_mtr_pool *));
6325         pools_mng->n = resize;
6326         pools_mng->pools = pools;
6327         if (old_pools)
6328                 mlx5_free(old_pools);
6329         return 0;
6330 }
6331
6332 /**
6333  * Prepare a new meter and/or a new meter pool.
6334  *
6335  * @param[in] dev
6336  *   Pointer to the Ethernet device structure.
6337  * @param[out] mtr_free
6338  *   Where to put the pointer of a new meter.g.
6339  *
6340  * @return
6341  *   The meter pool pointer and @mtr_free is set on success,
6342  *   NULL otherwise and rte_errno is set.
6343  */
6344 static struct mlx5_aso_mtr_pool *
6345 flow_dv_mtr_pool_create(struct rte_eth_dev *dev,
6346                              struct mlx5_aso_mtr **mtr_free)
6347 {
6348         struct mlx5_priv *priv = dev->data->dev_private;
6349         struct mlx5_aso_mtr_pools_mng *pools_mng =
6350                                 &priv->sh->mtrmng->pools_mng;
6351         struct mlx5_aso_mtr_pool *pool = NULL;
6352         struct mlx5_devx_obj *dcs = NULL;
6353         uint32_t i;
6354         uint32_t log_obj_size;
6355
6356         log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
6357         dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx,
6358                         priv->sh->pdn, log_obj_size);
6359         if (!dcs) {
6360                 rte_errno = ENODATA;
6361                 return NULL;
6362         }
6363         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
6364         if (!pool) {
6365                 rte_errno = ENOMEM;
6366                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6367                 return NULL;
6368         }
6369         pool->devx_obj = dcs;
6370         pool->index = pools_mng->n_valid;
6371         if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) {
6372                 mlx5_free(pool);
6373                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6374                 return NULL;
6375         }
6376         pools_mng->pools[pool->index] = pool;
6377         pools_mng->n_valid++;
6378         for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) {
6379                 pool->mtrs[i].offset = i;
6380                 LIST_INSERT_HEAD(&pools_mng->meters,
6381                                                 &pool->mtrs[i], next);
6382         }
6383         pool->mtrs[0].offset = 0;
6384         *mtr_free = &pool->mtrs[0];
6385         return pool;
6386 }
6387
6388 /**
6389  * Release a flow meter into pool.
6390  *
6391  * @param[in] dev
6392  *   Pointer to the Ethernet device structure.
6393  * @param[in] mtr_idx
6394  *   Index to aso flow meter.
6395  */
6396 static void
6397 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx)
6398 {
6399         struct mlx5_priv *priv = dev->data->dev_private;
6400         struct mlx5_aso_mtr_pools_mng *pools_mng =
6401                                 &priv->sh->mtrmng->pools_mng;
6402         struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
6403
6404         MLX5_ASSERT(aso_mtr);
6405         rte_spinlock_lock(&pools_mng->mtrsl);
6406         memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info));
6407         aso_mtr->state = ASO_METER_FREE;
6408         LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next);
6409         rte_spinlock_unlock(&pools_mng->mtrsl);
6410 }
6411
6412 /**
6413  * Allocate a aso flow meter.
6414  *
6415  * @param[in] dev
6416  *   Pointer to the Ethernet device structure.
6417  *
6418  * @return
6419  *   Index to aso flow meter on success, 0 otherwise and rte_errno is set.
6420  */
6421 static uint32_t
6422 flow_dv_mtr_alloc(struct rte_eth_dev *dev)
6423 {
6424         struct mlx5_priv *priv = dev->data->dev_private;
6425         struct mlx5_aso_mtr *mtr_free = NULL;
6426         struct mlx5_aso_mtr_pools_mng *pools_mng =
6427                                 &priv->sh->mtrmng->pools_mng;
6428         struct mlx5_aso_mtr_pool *pool;
6429         uint32_t mtr_idx = 0;
6430
6431         if (!priv->config.devx) {
6432                 rte_errno = ENOTSUP;
6433                 return 0;
6434         }
6435         /* Allocate the flow meter memory. */
6436         /* Get free meters from management. */
6437         rte_spinlock_lock(&pools_mng->mtrsl);
6438         mtr_free = LIST_FIRST(&pools_mng->meters);
6439         if (mtr_free)
6440                 LIST_REMOVE(mtr_free, next);
6441         if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) {
6442                 rte_spinlock_unlock(&pools_mng->mtrsl);
6443                 return 0;
6444         }
6445         mtr_free->state = ASO_METER_WAIT;
6446         rte_spinlock_unlock(&pools_mng->mtrsl);
6447         pool = container_of(mtr_free,
6448                         struct mlx5_aso_mtr_pool,
6449                         mtrs[mtr_free->offset]);
6450         mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
6451         if (!mtr_free->fm.meter_action) {
6452 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
6453                 struct rte_flow_error error;
6454                 uint8_t reg_id;
6455
6456                 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
6457                 mtr_free->fm.meter_action =
6458                         mlx5_glue->dv_create_flow_action_aso
6459                                                 (priv->sh->rx_domain,
6460                                                  pool->devx_obj->obj,
6461                                                  mtr_free->offset,
6462                                                  (1 << MLX5_FLOW_COLOR_GREEN),
6463                                                  reg_id - REG_C_0);
6464 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
6465                 if (!mtr_free->fm.meter_action) {
6466                         flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
6467                         return 0;
6468                 }
6469         }
6470         return mtr_idx;
6471 }
6472
6473 /**
6474  * Verify the @p attributes will be correctly understood by the NIC and store
6475  * them in the @p flow if everything is correct.
6476  *
6477  * @param[in] dev
6478  *   Pointer to dev struct.
6479  * @param[in] attributes
6480  *   Pointer to flow attributes
6481  * @param[in] external
6482  *   This flow rule is created by request external to PMD.
6483  * @param[out] error
6484  *   Pointer to error structure.
6485  *
6486  * @return
6487  *   - 0 on success and non root table.
6488  *   - 1 on success and root table.
6489  *   - a negative errno value otherwise and rte_errno is set.
6490  */
6491 static int
6492 flow_dv_validate_attributes(struct rte_eth_dev *dev,
6493                             const struct mlx5_flow_tunnel *tunnel,
6494                             const struct rte_flow_attr *attributes,
6495                             const struct flow_grp_info *grp_info,
6496                             struct rte_flow_error *error)
6497 {
6498         struct mlx5_priv *priv = dev->data->dev_private;
6499         uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes);
6500         int ret = 0;
6501
6502 #ifndef HAVE_MLX5DV_DR
6503         RTE_SET_USED(tunnel);
6504         RTE_SET_USED(grp_info);
6505         if (attributes->group)
6506                 return rte_flow_error_set(error, ENOTSUP,
6507                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6508                                           NULL,
6509                                           "groups are not supported");
6510 #else
6511         uint32_t table = 0;
6512
6513         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
6514                                        grp_info, error);
6515         if (ret)
6516                 return ret;
6517         if (!table)
6518                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
6519 #endif
6520         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
6521             attributes->priority > lowest_priority)
6522                 return rte_flow_error_set(error, ENOTSUP,
6523                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
6524                                           NULL,
6525                                           "priority out of range");
6526         if (attributes->transfer) {
6527                 if (!priv->config.dv_esw_en)
6528                         return rte_flow_error_set
6529                                 (error, ENOTSUP,
6530                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6531                                  "E-Switch dr is not supported");
6532                 if (!(priv->representor || priv->master))
6533                         return rte_flow_error_set
6534                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6535                                  NULL, "E-Switch configuration can only be"
6536                                  " done by a master or a representor device");
6537                 if (attributes->egress)
6538                         return rte_flow_error_set
6539                                 (error, ENOTSUP,
6540                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
6541                                  "egress is not supported");
6542         }
6543         if (!(attributes->egress ^ attributes->ingress))
6544                 return rte_flow_error_set(error, ENOTSUP,
6545                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
6546                                           "must specify exactly one of "
6547                                           "ingress or egress");
6548         return ret;
6549 }
6550
6551 static uint16_t
6552 mlx5_flow_locate_proto_l3(const struct rte_flow_item **head,
6553                           const struct rte_flow_item *end)
6554 {
6555         const struct rte_flow_item *item = *head;
6556         uint16_t l3_protocol;
6557
6558         for (; item != end; item++) {
6559                 switch (item->type) {
6560                 default:
6561                         break;
6562                 case RTE_FLOW_ITEM_TYPE_IPV4:
6563                         l3_protocol = RTE_ETHER_TYPE_IPV4;
6564                         goto l3_ok;
6565                 case RTE_FLOW_ITEM_TYPE_IPV6:
6566                         l3_protocol = RTE_ETHER_TYPE_IPV6;
6567                         goto l3_ok;
6568                 case RTE_FLOW_ITEM_TYPE_ETH:
6569                         if (item->mask && item->spec) {
6570                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_eth,
6571                                                             type, item,
6572                                                             l3_protocol);
6573                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6574                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6575                                         goto l3_ok;
6576                         }
6577                         break;
6578                 case RTE_FLOW_ITEM_TYPE_VLAN:
6579                         if (item->mask && item->spec) {
6580                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_vlan,
6581                                                             inner_type, item,
6582                                                             l3_protocol);
6583                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6584                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6585                                         goto l3_ok;
6586                         }
6587                         break;
6588                 }
6589         }
6590         return 0;
6591 l3_ok:
6592         *head = item;
6593         return l3_protocol;
6594 }
6595
6596 static uint8_t
6597 mlx5_flow_locate_proto_l4(const struct rte_flow_item **head,
6598                           const struct rte_flow_item *end)
6599 {
6600         const struct rte_flow_item *item = *head;
6601         uint8_t l4_protocol;
6602
6603         for (; item != end; item++) {
6604                 switch (item->type) {
6605                 default:
6606                         break;
6607                 case RTE_FLOW_ITEM_TYPE_TCP:
6608                         l4_protocol = IPPROTO_TCP;
6609                         goto l4_ok;
6610                 case RTE_FLOW_ITEM_TYPE_UDP:
6611                         l4_protocol = IPPROTO_UDP;
6612                         goto l4_ok;
6613                 case RTE_FLOW_ITEM_TYPE_IPV4:
6614                         if (item->mask && item->spec) {
6615                                 const struct rte_flow_item_ipv4 *mask, *spec;
6616
6617                                 mask = (typeof(mask))item->mask;
6618                                 spec = (typeof(spec))item->spec;
6619                                 l4_protocol = mask->hdr.next_proto_id &
6620                                               spec->hdr.next_proto_id;
6621                                 if (l4_protocol == IPPROTO_TCP ||
6622                                     l4_protocol == IPPROTO_UDP)
6623                                         goto l4_ok;
6624                         }
6625                         break;
6626                 case RTE_FLOW_ITEM_TYPE_IPV6:
6627                         if (item->mask && item->spec) {
6628                                 const struct rte_flow_item_ipv6 *mask, *spec;
6629                                 mask = (typeof(mask))item->mask;
6630                                 spec = (typeof(spec))item->spec;
6631                                 l4_protocol = mask->hdr.proto & spec->hdr.proto;
6632                                 if (l4_protocol == IPPROTO_TCP ||
6633                                     l4_protocol == IPPROTO_UDP)
6634                                         goto l4_ok;
6635                         }
6636                         break;
6637                 }
6638         }
6639         return 0;
6640 l4_ok:
6641         *head = item;
6642         return l4_protocol;
6643 }
6644
6645 static int
6646 flow_dv_validate_item_integrity(struct rte_eth_dev *dev,
6647                                 const struct rte_flow_item *rule_items,
6648                                 const struct rte_flow_item *integrity_item,
6649                                 struct rte_flow_error *error)
6650 {
6651         struct mlx5_priv *priv = dev->data->dev_private;
6652         const struct rte_flow_item *tunnel_item, *end_item, *item = rule_items;
6653         const struct rte_flow_item_integrity *mask = (typeof(mask))
6654                                                      integrity_item->mask;
6655         const struct rte_flow_item_integrity *spec = (typeof(spec))
6656                                                      integrity_item->spec;
6657         uint32_t protocol;
6658
6659         if (!priv->config.hca_attr.pkt_integrity_match)
6660                 return rte_flow_error_set(error, ENOTSUP,
6661                                           RTE_FLOW_ERROR_TYPE_ITEM,
6662                                           integrity_item,
6663                                           "packet integrity integrity_item not supported");
6664         if (!mask)
6665                 mask = &rte_flow_item_integrity_mask;
6666         if (!mlx5_validate_integrity_item(mask))
6667                 return rte_flow_error_set(error, ENOTSUP,
6668                                           RTE_FLOW_ERROR_TYPE_ITEM,
6669                                           integrity_item,
6670                                           "unsupported integrity filter");
6671         tunnel_item = mlx5_flow_find_tunnel_item(rule_items);
6672         if (spec->level > 1) {
6673                 if (!tunnel_item)
6674                         return rte_flow_error_set(error, ENOTSUP,
6675                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6676                                                   integrity_item,
6677                                                   "missing tunnel item");
6678                 item = tunnel_item;
6679                 end_item = mlx5_find_end_item(tunnel_item);
6680         } else {
6681                 end_item = tunnel_item ? tunnel_item :
6682                            mlx5_find_end_item(integrity_item);
6683         }
6684         if (mask->l3_ok || mask->ipv4_csum_ok) {
6685                 protocol = mlx5_flow_locate_proto_l3(&item, end_item);
6686                 if (!protocol)
6687                         return rte_flow_error_set(error, EINVAL,
6688                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6689                                                   integrity_item,
6690                                                   "missing L3 protocol");
6691         }
6692         if (mask->l4_ok || mask->l4_csum_ok) {
6693                 protocol = mlx5_flow_locate_proto_l4(&item, end_item);
6694                 if (!protocol)
6695                         return rte_flow_error_set(error, EINVAL,
6696                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6697                                                   integrity_item,
6698                                                   "missing L4 protocol");
6699         }
6700         return 0;
6701 }
6702
6703 /**
6704  * Internal validation function. For validating both actions and items.
6705  *
6706  * @param[in] dev
6707  *   Pointer to the rte_eth_dev structure.
6708  * @param[in] attr
6709  *   Pointer to the flow attributes.
6710  * @param[in] items
6711  *   Pointer to the list of items.
6712  * @param[in] actions
6713  *   Pointer to the list of actions.
6714  * @param[in] external
6715  *   This flow rule is created by request external to PMD.
6716  * @param[in] hairpin
6717  *   Number of hairpin TX actions, 0 means classic flow.
6718  * @param[out] error
6719  *   Pointer to the error structure.
6720  *
6721  * @return
6722  *   0 on success, a negative errno value otherwise and rte_errno is set.
6723  */
6724 static int
6725 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
6726                  const struct rte_flow_item items[],
6727                  const struct rte_flow_action actions[],
6728                  bool external, int hairpin, struct rte_flow_error *error)
6729 {
6730         int ret;
6731         uint64_t action_flags = 0;
6732         uint64_t item_flags = 0;
6733         uint64_t last_item = 0;
6734         uint8_t next_protocol = 0xff;
6735         uint16_t ether_type = 0;
6736         int actions_n = 0;
6737         uint8_t item_ipv6_proto = 0;
6738         int fdb_mirror_limit = 0;
6739         int modify_after_mirror = 0;
6740         const struct rte_flow_item *geneve_item = NULL;
6741         const struct rte_flow_item *gre_item = NULL;
6742         const struct rte_flow_item *gtp_item = NULL;
6743         const struct rte_flow_action_raw_decap *decap;
6744         const struct rte_flow_action_raw_encap *encap;
6745         const struct rte_flow_action_rss *rss = NULL;
6746         const struct rte_flow_action_rss *sample_rss = NULL;
6747         const struct rte_flow_action_count *sample_count = NULL;
6748         const struct rte_flow_item_tcp nic_tcp_mask = {
6749                 .hdr = {
6750                         .tcp_flags = 0xFF,
6751                         .src_port = RTE_BE16(UINT16_MAX),
6752                         .dst_port = RTE_BE16(UINT16_MAX),
6753                 }
6754         };
6755         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
6756                 .hdr = {
6757                         .src_addr =
6758                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6759                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6760                         .dst_addr =
6761                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6762                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6763                         .vtc_flow = RTE_BE32(0xffffffff),
6764                         .proto = 0xff,
6765                         .hop_limits = 0xff,
6766                 },
6767                 .has_frag_ext = 1,
6768         };
6769         const struct rte_flow_item_ecpri nic_ecpri_mask = {
6770                 .hdr = {
6771                         .common = {
6772                                 .u32 =
6773                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
6774                                         .type = 0xFF,
6775                                         }).u32),
6776                         },
6777                         .dummy[0] = 0xffffffff,
6778                 },
6779         };
6780         struct mlx5_priv *priv = dev->data->dev_private;
6781         struct mlx5_dev_config *dev_conf = &priv->config;
6782         uint16_t queue_index = 0xFFFF;
6783         const struct rte_flow_item_vlan *vlan_m = NULL;
6784         uint32_t rw_act_num = 0;
6785         uint64_t is_root;
6786         const struct mlx5_flow_tunnel *tunnel;
6787         enum mlx5_tof_rule_type tof_rule_type;
6788         struct flow_grp_info grp_info = {
6789                 .external = !!external,
6790                 .transfer = !!attr->transfer,
6791                 .fdb_def_rule = !!priv->fdb_def_rule,
6792                 .std_tbl_fix = true,
6793         };
6794         const struct rte_eth_hairpin_conf *conf;
6795         const struct rte_flow_item *rule_items = items;
6796         const struct rte_flow_item *port_id_item = NULL;
6797         bool def_policy = false;
6798
6799         if (items == NULL)
6800                 return -1;
6801         tunnel = is_tunnel_offload_active(dev) ?
6802                  mlx5_get_tof(items, actions, &tof_rule_type) : NULL;
6803         if (tunnel) {
6804                 if (priv->representor)
6805                         return rte_flow_error_set
6806                                 (error, ENOTSUP,
6807                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6808                                  NULL, "decap not supported for VF representor");
6809                 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE)
6810                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6811                 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE)
6812                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
6813                                         MLX5_FLOW_ACTION_DECAP;
6814                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
6815                                         (dev, attr, tunnel, tof_rule_type);
6816         }
6817         ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error);
6818         if (ret < 0)
6819                 return ret;
6820         is_root = (uint64_t)ret;
6821         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6822                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6823                 int type = items->type;
6824
6825                 if (!mlx5_flow_os_item_supported(type))
6826                         return rte_flow_error_set(error, ENOTSUP,
6827                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6828                                                   NULL, "item not supported");
6829                 switch (type) {
6830                 case RTE_FLOW_ITEM_TYPE_VOID:
6831                         break;
6832                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6833                         ret = flow_dv_validate_item_port_id
6834                                         (dev, items, attr, item_flags, error);
6835                         if (ret < 0)
6836                                 return ret;
6837                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6838                         port_id_item = items;
6839                         break;
6840                 case RTE_FLOW_ITEM_TYPE_ETH:
6841                         ret = mlx5_flow_validate_item_eth(items, item_flags,
6842                                                           true, error);
6843                         if (ret < 0)
6844                                 return ret;
6845                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
6846                                              MLX5_FLOW_LAYER_OUTER_L2;
6847                         if (items->mask != NULL && items->spec != NULL) {
6848                                 ether_type =
6849                                         ((const struct rte_flow_item_eth *)
6850                                          items->spec)->type;
6851                                 ether_type &=
6852                                         ((const struct rte_flow_item_eth *)
6853                                          items->mask)->type;
6854                                 ether_type = rte_be_to_cpu_16(ether_type);
6855                         } else {
6856                                 ether_type = 0;
6857                         }
6858                         break;
6859                 case RTE_FLOW_ITEM_TYPE_VLAN:
6860                         ret = flow_dv_validate_item_vlan(items, item_flags,
6861                                                          dev, error);
6862                         if (ret < 0)
6863                                 return ret;
6864                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
6865                                              MLX5_FLOW_LAYER_OUTER_VLAN;
6866                         if (items->mask != NULL && items->spec != NULL) {
6867                                 ether_type =
6868                                         ((const struct rte_flow_item_vlan *)
6869                                          items->spec)->inner_type;
6870                                 ether_type &=
6871                                         ((const struct rte_flow_item_vlan *)
6872                                          items->mask)->inner_type;
6873                                 ether_type = rte_be_to_cpu_16(ether_type);
6874                         } else {
6875                                 ether_type = 0;
6876                         }
6877                         /* Store outer VLAN mask for of_push_vlan action. */
6878                         if (!tunnel)
6879                                 vlan_m = items->mask;
6880                         break;
6881                 case RTE_FLOW_ITEM_TYPE_IPV4:
6882                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6883                                                   &item_flags, &tunnel);
6884                         ret = flow_dv_validate_item_ipv4(items, item_flags,
6885                                                          last_item, ether_type,
6886                                                          error);
6887                         if (ret < 0)
6888                                 return ret;
6889                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
6890                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
6891                         if (items->mask != NULL &&
6892                             ((const struct rte_flow_item_ipv4 *)
6893                              items->mask)->hdr.next_proto_id) {
6894                                 next_protocol =
6895                                         ((const struct rte_flow_item_ipv4 *)
6896                                          (items->spec))->hdr.next_proto_id;
6897                                 next_protocol &=
6898                                         ((const struct rte_flow_item_ipv4 *)
6899                                          (items->mask))->hdr.next_proto_id;
6900                         } else {
6901                                 /* Reset for inner layer. */
6902                                 next_protocol = 0xff;
6903                         }
6904                         break;
6905                 case RTE_FLOW_ITEM_TYPE_IPV6:
6906                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6907                                                   &item_flags, &tunnel);
6908                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
6909                                                            last_item,
6910                                                            ether_type,
6911                                                            &nic_ipv6_mask,
6912                                                            error);
6913                         if (ret < 0)
6914                                 return ret;
6915                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
6916                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
6917                         if (items->mask != NULL &&
6918                             ((const struct rte_flow_item_ipv6 *)
6919                              items->mask)->hdr.proto) {
6920                                 item_ipv6_proto =
6921                                         ((const struct rte_flow_item_ipv6 *)
6922                                          items->spec)->hdr.proto;
6923                                 next_protocol =
6924                                         ((const struct rte_flow_item_ipv6 *)
6925                                          items->spec)->hdr.proto;
6926                                 next_protocol &=
6927                                         ((const struct rte_flow_item_ipv6 *)
6928                                          items->mask)->hdr.proto;
6929                         } else {
6930                                 /* Reset for inner layer. */
6931                                 next_protocol = 0xff;
6932                         }
6933                         break;
6934                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
6935                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
6936                                                                   item_flags,
6937                                                                   error);
6938                         if (ret < 0)
6939                                 return ret;
6940                         last_item = tunnel ?
6941                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
6942                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
6943                         if (items->mask != NULL &&
6944                             ((const struct rte_flow_item_ipv6_frag_ext *)
6945                              items->mask)->hdr.next_header) {
6946                                 next_protocol =
6947                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6948                                  items->spec)->hdr.next_header;
6949                                 next_protocol &=
6950                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6951                                  items->mask)->hdr.next_header;
6952                         } else {
6953                                 /* Reset for inner layer. */
6954                                 next_protocol = 0xff;
6955                         }
6956                         break;
6957                 case RTE_FLOW_ITEM_TYPE_TCP:
6958                         ret = mlx5_flow_validate_item_tcp
6959                                                 (items, item_flags,
6960                                                  next_protocol,
6961                                                  &nic_tcp_mask,
6962                                                  error);
6963                         if (ret < 0)
6964                                 return ret;
6965                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
6966                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
6967                         break;
6968                 case RTE_FLOW_ITEM_TYPE_UDP:
6969                         ret = mlx5_flow_validate_item_udp(items, item_flags,
6970                                                           next_protocol,
6971                                                           error);
6972                         if (ret < 0)
6973                                 return ret;
6974                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
6975                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
6976                         break;
6977                 case RTE_FLOW_ITEM_TYPE_GRE:
6978                         ret = mlx5_flow_validate_item_gre(items, item_flags,
6979                                                           next_protocol, error);
6980                         if (ret < 0)
6981                                 return ret;
6982                         gre_item = items;
6983                         last_item = MLX5_FLOW_LAYER_GRE;
6984                         break;
6985                 case RTE_FLOW_ITEM_TYPE_NVGRE:
6986                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
6987                                                             next_protocol,
6988                                                             error);
6989                         if (ret < 0)
6990                                 return ret;
6991                         last_item = MLX5_FLOW_LAYER_NVGRE;
6992                         break;
6993                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
6994                         ret = mlx5_flow_validate_item_gre_key
6995                                 (items, item_flags, gre_item, error);
6996                         if (ret < 0)
6997                                 return ret;
6998                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
6999                         break;
7000                 case RTE_FLOW_ITEM_TYPE_VXLAN:
7001                         ret = mlx5_flow_validate_item_vxlan(dev, items,
7002                                                             item_flags, attr,
7003                                                             error);
7004                         if (ret < 0)
7005                                 return ret;
7006                         last_item = MLX5_FLOW_LAYER_VXLAN;
7007                         break;
7008                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
7009                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
7010                                                                 item_flags, dev,
7011                                                                 error);
7012                         if (ret < 0)
7013                                 return ret;
7014                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
7015                         break;
7016                 case RTE_FLOW_ITEM_TYPE_GENEVE:
7017                         ret = mlx5_flow_validate_item_geneve(items,
7018                                                              item_flags, dev,
7019                                                              error);
7020                         if (ret < 0)
7021                                 return ret;
7022                         geneve_item = items;
7023                         last_item = MLX5_FLOW_LAYER_GENEVE;
7024                         break;
7025                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
7026                         ret = mlx5_flow_validate_item_geneve_opt(items,
7027                                                                  last_item,
7028                                                                  geneve_item,
7029                                                                  dev,
7030                                                                  error);
7031                         if (ret < 0)
7032                                 return ret;
7033                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
7034                         break;
7035                 case RTE_FLOW_ITEM_TYPE_MPLS:
7036                         ret = mlx5_flow_validate_item_mpls(dev, items,
7037                                                            item_flags,
7038                                                            last_item, error);
7039                         if (ret < 0)
7040                                 return ret;
7041                         last_item = MLX5_FLOW_LAYER_MPLS;
7042                         break;
7043
7044                 case RTE_FLOW_ITEM_TYPE_MARK:
7045                         ret = flow_dv_validate_item_mark(dev, items, attr,
7046                                                          error);
7047                         if (ret < 0)
7048                                 return ret;
7049                         last_item = MLX5_FLOW_ITEM_MARK;
7050                         break;
7051                 case RTE_FLOW_ITEM_TYPE_META:
7052                         ret = flow_dv_validate_item_meta(dev, items, attr,
7053                                                          error);
7054                         if (ret < 0)
7055                                 return ret;
7056                         last_item = MLX5_FLOW_ITEM_METADATA;
7057                         break;
7058                 case RTE_FLOW_ITEM_TYPE_ICMP:
7059                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
7060                                                            next_protocol,
7061                                                            error);
7062                         if (ret < 0)
7063                                 return ret;
7064                         last_item = MLX5_FLOW_LAYER_ICMP;
7065                         break;
7066                 case RTE_FLOW_ITEM_TYPE_ICMP6:
7067                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
7068                                                             next_protocol,
7069                                                             error);
7070                         if (ret < 0)
7071                                 return ret;
7072                         item_ipv6_proto = IPPROTO_ICMPV6;
7073                         last_item = MLX5_FLOW_LAYER_ICMP6;
7074                         break;
7075                 case RTE_FLOW_ITEM_TYPE_TAG:
7076                         ret = flow_dv_validate_item_tag(dev, items,
7077                                                         attr, error);
7078                         if (ret < 0)
7079                                 return ret;
7080                         last_item = MLX5_FLOW_ITEM_TAG;
7081                         break;
7082                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
7083                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
7084                         break;
7085                 case RTE_FLOW_ITEM_TYPE_GTP:
7086                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
7087                                                         error);
7088                         if (ret < 0)
7089                                 return ret;
7090                         gtp_item = items;
7091                         last_item = MLX5_FLOW_LAYER_GTP;
7092                         break;
7093                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
7094                         ret = flow_dv_validate_item_gtp_psc(items, last_item,
7095                                                             gtp_item, attr,
7096                                                             error);
7097                         if (ret < 0)
7098                                 return ret;
7099                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
7100                         break;
7101                 case RTE_FLOW_ITEM_TYPE_ECPRI:
7102                         /* Capacity will be checked in the translate stage. */
7103                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
7104                                                             last_item,
7105                                                             ether_type,
7106                                                             &nic_ecpri_mask,
7107                                                             error);
7108                         if (ret < 0)
7109                                 return ret;
7110                         last_item = MLX5_FLOW_LAYER_ECPRI;
7111                         break;
7112                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
7113                         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY)
7114                                 return rte_flow_error_set
7115                                         (error, ENOTSUP,
7116                                          RTE_FLOW_ERROR_TYPE_ITEM,
7117                                          NULL, "multiple integrity items not supported");
7118                         ret = flow_dv_validate_item_integrity(dev, rule_items,
7119                                                               items, error);
7120                         if (ret < 0)
7121                                 return ret;
7122                         last_item = MLX5_FLOW_ITEM_INTEGRITY;
7123                         break;
7124                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
7125                         ret = flow_dv_validate_item_aso_ct(dev, items,
7126                                                            &item_flags, error);
7127                         if (ret < 0)
7128                                 return ret;
7129                         break;
7130                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
7131                         /* tunnel offload item was processed before
7132                          * list it here as a supported type
7133                          */
7134                         break;
7135                 default:
7136                         return rte_flow_error_set(error, ENOTSUP,
7137                                                   RTE_FLOW_ERROR_TYPE_ITEM,
7138                                                   NULL, "item not supported");
7139                 }
7140                 item_flags |= last_item;
7141         }
7142         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7143                 int type = actions->type;
7144                 bool shared_count = false;
7145
7146                 if (!mlx5_flow_os_action_supported(type))
7147                         return rte_flow_error_set(error, ENOTSUP,
7148                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7149                                                   actions,
7150                                                   "action not supported");
7151                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
7152                         return rte_flow_error_set(error, ENOTSUP,
7153                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7154                                                   actions, "too many actions");
7155                 if (action_flags &
7156                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
7157                         return rte_flow_error_set(error, ENOTSUP,
7158                                 RTE_FLOW_ERROR_TYPE_ACTION,
7159                                 NULL, "meter action with policy "
7160                                 "must be the last action");
7161                 switch (type) {
7162                 case RTE_FLOW_ACTION_TYPE_VOID:
7163                         break;
7164                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7165                         ret = flow_dv_validate_action_port_id(dev,
7166                                                               action_flags,
7167                                                               actions,
7168                                                               attr,
7169                                                               error);
7170                         if (ret)
7171                                 return ret;
7172                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7173                         ++actions_n;
7174                         break;
7175                 case RTE_FLOW_ACTION_TYPE_FLAG:
7176                         ret = flow_dv_validate_action_flag(dev, action_flags,
7177                                                            attr, error);
7178                         if (ret < 0)
7179                                 return ret;
7180                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7181                                 /* Count all modify-header actions as one. */
7182                                 if (!(action_flags &
7183                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7184                                         ++actions_n;
7185                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
7186                                                 MLX5_FLOW_ACTION_MARK_EXT;
7187                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7188                                         modify_after_mirror = 1;
7189
7190                         } else {
7191                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
7192                                 ++actions_n;
7193                         }
7194                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7195                         break;
7196                 case RTE_FLOW_ACTION_TYPE_MARK:
7197                         ret = flow_dv_validate_action_mark(dev, actions,
7198                                                            action_flags,
7199                                                            attr, error);
7200                         if (ret < 0)
7201                                 return ret;
7202                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7203                                 /* Count all modify-header actions as one. */
7204                                 if (!(action_flags &
7205                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7206                                         ++actions_n;
7207                                 action_flags |= MLX5_FLOW_ACTION_MARK |
7208                                                 MLX5_FLOW_ACTION_MARK_EXT;
7209                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7210                                         modify_after_mirror = 1;
7211                         } else {
7212                                 action_flags |= MLX5_FLOW_ACTION_MARK;
7213                                 ++actions_n;
7214                         }
7215                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7216                         break;
7217                 case RTE_FLOW_ACTION_TYPE_SET_META:
7218                         ret = flow_dv_validate_action_set_meta(dev, actions,
7219                                                                action_flags,
7220                                                                attr, error);
7221                         if (ret < 0)
7222                                 return ret;
7223                         /* Count all modify-header actions as one action. */
7224                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7225                                 ++actions_n;
7226                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7227                                 modify_after_mirror = 1;
7228                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7229                         rw_act_num += MLX5_ACT_NUM_SET_META;
7230                         break;
7231                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7232                         ret = flow_dv_validate_action_set_tag(dev, actions,
7233                                                               action_flags,
7234                                                               attr, error);
7235                         if (ret < 0)
7236                                 return ret;
7237                         /* Count all modify-header actions as one action. */
7238                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7239                                 ++actions_n;
7240                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7241                                 modify_after_mirror = 1;
7242                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7243                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7244                         break;
7245                 case RTE_FLOW_ACTION_TYPE_DROP:
7246                         ret = mlx5_flow_validate_action_drop(action_flags,
7247                                                              attr, error);
7248                         if (ret < 0)
7249                                 return ret;
7250                         action_flags |= MLX5_FLOW_ACTION_DROP;
7251                         ++actions_n;
7252                         break;
7253                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7254                         ret = mlx5_flow_validate_action_queue(actions,
7255                                                               action_flags, dev,
7256                                                               attr, error);
7257                         if (ret < 0)
7258                                 return ret;
7259                         queue_index = ((const struct rte_flow_action_queue *)
7260                                                         (actions->conf))->index;
7261                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7262                         ++actions_n;
7263                         break;
7264                 case RTE_FLOW_ACTION_TYPE_RSS:
7265                         rss = actions->conf;
7266                         ret = mlx5_flow_validate_action_rss(actions,
7267                                                             action_flags, dev,
7268                                                             attr, item_flags,
7269                                                             error);
7270                         if (ret < 0)
7271                                 return ret;
7272                         if (rss && sample_rss &&
7273                             (sample_rss->level != rss->level ||
7274                             sample_rss->types != rss->types))
7275                                 return rte_flow_error_set(error, ENOTSUP,
7276                                         RTE_FLOW_ERROR_TYPE_ACTION,
7277                                         NULL,
7278                                         "Can't use the different RSS types "
7279                                         "or level in the same flow");
7280                         if (rss != NULL && rss->queue_num)
7281                                 queue_index = rss->queue[0];
7282                         action_flags |= MLX5_FLOW_ACTION_RSS;
7283                         ++actions_n;
7284                         break;
7285                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
7286                         ret =
7287                         mlx5_flow_validate_action_default_miss(action_flags,
7288                                         attr, error);
7289                         if (ret < 0)
7290                                 return ret;
7291                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
7292                         ++actions_n;
7293                         break;
7294                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
7295                 case RTE_FLOW_ACTION_TYPE_COUNT:
7296                         shared_count = is_shared_action_count(actions);
7297                         ret = flow_dv_validate_action_count(dev, shared_count,
7298                                                             action_flags,
7299                                                             error);
7300                         if (ret < 0)
7301                                 return ret;
7302                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7303                         ++actions_n;
7304                         break;
7305                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7306                         if (flow_dv_validate_action_pop_vlan(dev,
7307                                                              action_flags,
7308                                                              actions,
7309                                                              item_flags, attr,
7310                                                              error))
7311                                 return -rte_errno;
7312                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7313                                 modify_after_mirror = 1;
7314                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7315                         ++actions_n;
7316                         break;
7317                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
7318                         ret = flow_dv_validate_action_push_vlan(dev,
7319                                                                 action_flags,
7320                                                                 vlan_m,
7321                                                                 actions, attr,
7322                                                                 error);
7323                         if (ret < 0)
7324                                 return ret;
7325                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7326                                 modify_after_mirror = 1;
7327                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
7328                         ++actions_n;
7329                         break;
7330                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
7331                         ret = flow_dv_validate_action_set_vlan_pcp
7332                                                 (action_flags, actions, error);
7333                         if (ret < 0)
7334                                 return ret;
7335                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7336                                 modify_after_mirror = 1;
7337                         /* Count PCP with push_vlan command. */
7338                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
7339                         break;
7340                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
7341                         ret = flow_dv_validate_action_set_vlan_vid
7342                                                 (item_flags, action_flags,
7343                                                  actions, error);
7344                         if (ret < 0)
7345                                 return ret;
7346                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7347                                 modify_after_mirror = 1;
7348                         /* Count VID with push_vlan command. */
7349                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
7350                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
7351                         break;
7352                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
7353                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
7354                         ret = flow_dv_validate_action_l2_encap(dev,
7355                                                                action_flags,
7356                                                                actions, attr,
7357                                                                error);
7358                         if (ret < 0)
7359                                 return ret;
7360                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
7361                         ++actions_n;
7362                         break;
7363                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
7364                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
7365                         ret = flow_dv_validate_action_decap(dev, action_flags,
7366                                                             actions, item_flags,
7367                                                             attr, error);
7368                         if (ret < 0)
7369                                 return ret;
7370                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7371                                 modify_after_mirror = 1;
7372                         action_flags |= MLX5_FLOW_ACTION_DECAP;
7373                         ++actions_n;
7374                         break;
7375                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
7376                         ret = flow_dv_validate_action_raw_encap_decap
7377                                 (dev, NULL, actions->conf, attr, &action_flags,
7378                                  &actions_n, actions, item_flags, error);
7379                         if (ret < 0)
7380                                 return ret;
7381                         break;
7382                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
7383                         decap = actions->conf;
7384                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
7385                                 ;
7386                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
7387                                 encap = NULL;
7388                                 actions--;
7389                         } else {
7390                                 encap = actions->conf;
7391                         }
7392                         ret = flow_dv_validate_action_raw_encap_decap
7393                                            (dev,
7394                                             decap ? decap : &empty_decap, encap,
7395                                             attr, &action_flags, &actions_n,
7396                                             actions, item_flags, error);
7397                         if (ret < 0)
7398                                 return ret;
7399                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7400                             (action_flags & MLX5_FLOW_ACTION_DECAP))
7401                                 modify_after_mirror = 1;
7402                         break;
7403                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
7404                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
7405                         ret = flow_dv_validate_action_modify_mac(action_flags,
7406                                                                  actions,
7407                                                                  item_flags,
7408                                                                  error);
7409                         if (ret < 0)
7410                                 return ret;
7411                         /* Count all modify-header actions as one action. */
7412                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7413                                 ++actions_n;
7414                         action_flags |= actions->type ==
7415                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
7416                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
7417                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
7418                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7419                                 modify_after_mirror = 1;
7420                         /*
7421                          * Even if the source and destination MAC addresses have
7422                          * overlap in the header with 4B alignment, the convert
7423                          * function will handle them separately and 4 SW actions
7424                          * will be created. And 2 actions will be added each
7425                          * time no matter how many bytes of address will be set.
7426                          */
7427                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
7428                         break;
7429                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
7430                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
7431                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
7432                                                                   actions,
7433                                                                   item_flags,
7434                                                                   error);
7435                         if (ret < 0)
7436                                 return ret;
7437                         /* Count all modify-header actions as one action. */
7438                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7439                                 ++actions_n;
7440                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7441                                 modify_after_mirror = 1;
7442                         action_flags |= actions->type ==
7443                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
7444                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
7445                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
7446                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
7447                         break;
7448                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
7449                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
7450                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
7451                                                                   actions,
7452                                                                   item_flags,
7453                                                                   error);
7454                         if (ret < 0)
7455                                 return ret;
7456                         if (item_ipv6_proto == IPPROTO_ICMPV6)
7457                                 return rte_flow_error_set(error, ENOTSUP,
7458                                         RTE_FLOW_ERROR_TYPE_ACTION,
7459                                         actions,
7460                                         "Can't change header "
7461                                         "with ICMPv6 proto");
7462                         /* Count all modify-header actions as one action. */
7463                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7464                                 ++actions_n;
7465                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7466                                 modify_after_mirror = 1;
7467                         action_flags |= actions->type ==
7468                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
7469                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
7470                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
7471                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
7472                         break;
7473                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
7474                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
7475                         ret = flow_dv_validate_action_modify_tp(action_flags,
7476                                                                 actions,
7477                                                                 item_flags,
7478                                                                 error);
7479                         if (ret < 0)
7480                                 return ret;
7481                         /* Count all modify-header actions as one action. */
7482                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7483                                 ++actions_n;
7484                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7485                                 modify_after_mirror = 1;
7486                         action_flags |= actions->type ==
7487                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
7488                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
7489                                                 MLX5_FLOW_ACTION_SET_TP_DST;
7490                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
7491                         break;
7492                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
7493                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
7494                         ret = flow_dv_validate_action_modify_ttl(action_flags,
7495                                                                  actions,
7496                                                                  item_flags,
7497                                                                  error);
7498                         if (ret < 0)
7499                                 return ret;
7500                         /* Count all modify-header actions as one action. */
7501                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7502                                 ++actions_n;
7503                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7504                                 modify_after_mirror = 1;
7505                         action_flags |= actions->type ==
7506                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
7507                                                 MLX5_FLOW_ACTION_SET_TTL :
7508                                                 MLX5_FLOW_ACTION_DEC_TTL;
7509                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
7510                         break;
7511                 case RTE_FLOW_ACTION_TYPE_JUMP:
7512                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
7513                                                            action_flags,
7514                                                            attr, external,
7515                                                            error);
7516                         if (ret)
7517                                 return ret;
7518                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7519                             fdb_mirror_limit)
7520                                 return rte_flow_error_set(error, EINVAL,
7521                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7522                                                   NULL,
7523                                                   "sample and jump action combination is not supported");
7524                         ++actions_n;
7525                         action_flags |= MLX5_FLOW_ACTION_JUMP;
7526                         break;
7527                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
7528                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
7529                         ret = flow_dv_validate_action_modify_tcp_seq
7530                                                                 (action_flags,
7531                                                                  actions,
7532                                                                  item_flags,
7533                                                                  error);
7534                         if (ret < 0)
7535                                 return ret;
7536                         /* Count all modify-header actions as one action. */
7537                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7538                                 ++actions_n;
7539                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7540                                 modify_after_mirror = 1;
7541                         action_flags |= actions->type ==
7542                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
7543                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
7544                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
7545                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
7546                         break;
7547                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
7548                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
7549                         ret = flow_dv_validate_action_modify_tcp_ack
7550                                                                 (action_flags,
7551                                                                  actions,
7552                                                                  item_flags,
7553                                                                  error);
7554                         if (ret < 0)
7555                                 return ret;
7556                         /* Count all modify-header actions as one action. */
7557                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7558                                 ++actions_n;
7559                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7560                                 modify_after_mirror = 1;
7561                         action_flags |= actions->type ==
7562                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
7563                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
7564                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
7565                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
7566                         break;
7567                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7568                         break;
7569                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
7570                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7571                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7572                         break;
7573                 case RTE_FLOW_ACTION_TYPE_METER:
7574                         ret = mlx5_flow_validate_action_meter(dev,
7575                                                               action_flags,
7576                                                               actions, attr,
7577                                                               port_id_item,
7578                                                               &def_policy,
7579                                                               error);
7580                         if (ret < 0)
7581                                 return ret;
7582                         action_flags |= MLX5_FLOW_ACTION_METER;
7583                         if (!def_policy)
7584                                 action_flags |=
7585                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
7586                         ++actions_n;
7587                         /* Meter action will add one more TAG action. */
7588                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7589                         break;
7590                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
7591                         if (!attr->transfer && !attr->group)
7592                                 return rte_flow_error_set(error, ENOTSUP,
7593                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7594                                                                            NULL,
7595                           "Shared ASO age action is not supported for group 0");
7596                         if (action_flags & MLX5_FLOW_ACTION_AGE)
7597                                 return rte_flow_error_set
7598                                                   (error, EINVAL,
7599                                                    RTE_FLOW_ERROR_TYPE_ACTION,
7600                                                    NULL,
7601                                                    "duplicate age actions set");
7602                         action_flags |= MLX5_FLOW_ACTION_AGE;
7603                         ++actions_n;
7604                         break;
7605                 case RTE_FLOW_ACTION_TYPE_AGE:
7606                         ret = flow_dv_validate_action_age(action_flags,
7607                                                           actions, dev,
7608                                                           error);
7609                         if (ret < 0)
7610                                 return ret;
7611                         /*
7612                          * Validate the regular AGE action (using counter)
7613                          * mutual exclusion with share counter actions.
7614                          */
7615                         if (!priv->sh->flow_hit_aso_en) {
7616                                 if (shared_count)
7617                                         return rte_flow_error_set
7618                                                 (error, EINVAL,
7619                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7620                                                 NULL,
7621                                                 "old age and shared count combination is not supported");
7622                                 if (sample_count)
7623                                         return rte_flow_error_set
7624                                                 (error, EINVAL,
7625                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7626                                                 NULL,
7627                                                 "old age action and count must be in the same sub flow");
7628                         }
7629                         action_flags |= MLX5_FLOW_ACTION_AGE;
7630                         ++actions_n;
7631                         break;
7632                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
7633                         ret = flow_dv_validate_action_modify_ipv4_dscp
7634                                                          (action_flags,
7635                                                           actions,
7636                                                           item_flags,
7637                                                           error);
7638                         if (ret < 0)
7639                                 return ret;
7640                         /* Count all modify-header actions as one action. */
7641                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7642                                 ++actions_n;
7643                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7644                                 modify_after_mirror = 1;
7645                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
7646                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7647                         break;
7648                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
7649                         ret = flow_dv_validate_action_modify_ipv6_dscp
7650                                                                 (action_flags,
7651                                                                  actions,
7652                                                                  item_flags,
7653                                                                  error);
7654                         if (ret < 0)
7655                                 return ret;
7656                         /* Count all modify-header actions as one action. */
7657                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7658                                 ++actions_n;
7659                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7660                                 modify_after_mirror = 1;
7661                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
7662                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7663                         break;
7664                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
7665                         ret = flow_dv_validate_action_sample(&action_flags,
7666                                                              actions, dev,
7667                                                              attr, item_flags,
7668                                                              rss, &sample_rss,
7669                                                              &sample_count,
7670                                                              &fdb_mirror_limit,
7671                                                              error);
7672                         if (ret < 0)
7673                                 return ret;
7674                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
7675                         ++actions_n;
7676                         break;
7677                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
7678                         ret = flow_dv_validate_action_modify_field(dev,
7679                                                                    action_flags,
7680                                                                    actions,
7681                                                                    attr,
7682                                                                    error);
7683                         if (ret < 0)
7684                                 return ret;
7685                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7686                                 modify_after_mirror = 1;
7687                         /* Count all modify-header actions as one action. */
7688                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7689                                 ++actions_n;
7690                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
7691                         rw_act_num += ret;
7692                         break;
7693                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
7694                         ret = flow_dv_validate_action_aso_ct(dev, action_flags,
7695                                                              item_flags, attr,
7696                                                              error);
7697                         if (ret < 0)
7698                                 return ret;
7699                         action_flags |= MLX5_FLOW_ACTION_CT;
7700                         break;
7701                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
7702                         /* tunnel offload action was processed before
7703                          * list it here as a supported type
7704                          */
7705                         break;
7706                 default:
7707                         return rte_flow_error_set(error, ENOTSUP,
7708                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7709                                                   actions,
7710                                                   "action not supported");
7711                 }
7712         }
7713         /*
7714          * Validate actions in flow rules
7715          * - Explicit decap action is prohibited by the tunnel offload API.
7716          * - Drop action in tunnel steer rule is prohibited by the API.
7717          * - Application cannot use MARK action because it's value can mask
7718          *   tunnel default miss nitification.
7719          * - JUMP in tunnel match rule has no support in current PMD
7720          *   implementation.
7721          * - TAG & META are reserved for future uses.
7722          */
7723         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
7724                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
7725                                             MLX5_FLOW_ACTION_MARK     |
7726                                             MLX5_FLOW_ACTION_SET_TAG  |
7727                                             MLX5_FLOW_ACTION_SET_META |
7728                                             MLX5_FLOW_ACTION_DROP;
7729
7730                 if (action_flags & bad_actions_mask)
7731                         return rte_flow_error_set
7732                                         (error, EINVAL,
7733                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7734                                         "Invalid RTE action in tunnel "
7735                                         "set decap rule");
7736                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
7737                         return rte_flow_error_set
7738                                         (error, EINVAL,
7739                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7740                                         "tunnel set decap rule must terminate "
7741                                         "with JUMP");
7742                 if (!attr->ingress)
7743                         return rte_flow_error_set
7744                                         (error, EINVAL,
7745                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7746                                         "tunnel flows for ingress traffic only");
7747         }
7748         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
7749                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
7750                                             MLX5_FLOW_ACTION_MARK    |
7751                                             MLX5_FLOW_ACTION_SET_TAG |
7752                                             MLX5_FLOW_ACTION_SET_META;
7753
7754                 if (action_flags & bad_actions_mask)
7755                         return rte_flow_error_set
7756                                         (error, EINVAL,
7757                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7758                                         "Invalid RTE action in tunnel "
7759                                         "set match rule");
7760         }
7761         /*
7762          * Validate the drop action mutual exclusion with other actions.
7763          * Drop action is mutually-exclusive with any other action, except for
7764          * Count action.
7765          * Drop action compatibility with tunnel offload was already validated.
7766          */
7767         if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH |
7768                             MLX5_FLOW_ACTION_TUNNEL_MATCH));
7769         else if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
7770             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
7771                 return rte_flow_error_set(error, EINVAL,
7772                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7773                                           "Drop action is mutually-exclusive "
7774                                           "with any other action, except for "
7775                                           "Count action");
7776         /* Eswitch has few restrictions on using items and actions */
7777         if (attr->transfer) {
7778                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7779                     action_flags & MLX5_FLOW_ACTION_FLAG)
7780                         return rte_flow_error_set(error, ENOTSUP,
7781                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7782                                                   NULL,
7783                                                   "unsupported action FLAG");
7784                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7785                     action_flags & MLX5_FLOW_ACTION_MARK)
7786                         return rte_flow_error_set(error, ENOTSUP,
7787                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7788                                                   NULL,
7789                                                   "unsupported action MARK");
7790                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
7791                         return rte_flow_error_set(error, ENOTSUP,
7792                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7793                                                   NULL,
7794                                                   "unsupported action QUEUE");
7795                 if (action_flags & MLX5_FLOW_ACTION_RSS)
7796                         return rte_flow_error_set(error, ENOTSUP,
7797                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7798                                                   NULL,
7799                                                   "unsupported action RSS");
7800                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
7801                         return rte_flow_error_set(error, EINVAL,
7802                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7803                                                   actions,
7804                                                   "no fate action is found");
7805         } else {
7806                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
7807                         return rte_flow_error_set(error, EINVAL,
7808                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7809                                                   actions,
7810                                                   "no fate action is found");
7811         }
7812         /*
7813          * Continue validation for Xcap and VLAN actions.
7814          * If hairpin is working in explicit TX rule mode, there is no actions
7815          * splitting and the validation of hairpin ingress flow should be the
7816          * same as other standard flows.
7817          */
7818         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
7819                              MLX5_FLOW_VLAN_ACTIONS)) &&
7820             (queue_index == 0xFFFF ||
7821              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN ||
7822              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
7823              conf->tx_explicit != 0))) {
7824                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
7825                     MLX5_FLOW_XCAP_ACTIONS)
7826                         return rte_flow_error_set(error, ENOTSUP,
7827                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7828                                                   NULL, "encap and decap "
7829                                                   "combination aren't supported");
7830                 if (!attr->transfer && attr->ingress) {
7831                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7832                                 return rte_flow_error_set
7833                                                 (error, ENOTSUP,
7834                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7835                                                  NULL, "encap is not supported"
7836                                                  " for ingress traffic");
7837                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7838                                 return rte_flow_error_set
7839                                                 (error, ENOTSUP,
7840                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7841                                                  NULL, "push VLAN action not "
7842                                                  "supported for ingress");
7843                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
7844                                         MLX5_FLOW_VLAN_ACTIONS)
7845                                 return rte_flow_error_set
7846                                                 (error, ENOTSUP,
7847                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7848                                                  NULL, "no support for "
7849                                                  "multiple VLAN actions");
7850                 }
7851         }
7852         if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) {
7853                 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS &
7854                         ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) &&
7855                         attr->ingress)
7856                         return rte_flow_error_set
7857                                 (error, ENOTSUP,
7858                                 RTE_FLOW_ERROR_TYPE_ACTION,
7859                                 NULL, "fate action not supported for "
7860                                 "meter with policy");
7861                 if (attr->egress) {
7862                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
7863                                 return rte_flow_error_set
7864                                         (error, ENOTSUP,
7865                                         RTE_FLOW_ERROR_TYPE_ACTION,
7866                                         NULL, "modify header action in egress "
7867                                         "cannot be done before meter action");
7868                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7869                                 return rte_flow_error_set
7870                                         (error, ENOTSUP,
7871                                         RTE_FLOW_ERROR_TYPE_ACTION,
7872                                         NULL, "encap action in egress "
7873                                         "cannot be done before meter action");
7874                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7875                                 return rte_flow_error_set
7876                                         (error, ENOTSUP,
7877                                         RTE_FLOW_ERROR_TYPE_ACTION,
7878                                         NULL, "push vlan action in egress "
7879                                         "cannot be done before meter action");
7880                 }
7881         }
7882         /*
7883          * Hairpin flow will add one more TAG action in TX implicit mode.
7884          * In TX explicit mode, there will be no hairpin flow ID.
7885          */
7886         if (hairpin > 0)
7887                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7888         /* extra metadata enabled: one more TAG action will be add. */
7889         if (dev_conf->dv_flow_en &&
7890             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
7891             mlx5_flow_ext_mreg_supported(dev))
7892                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7893         if (rw_act_num >
7894                         flow_dv_modify_hdr_action_max(dev, is_root)) {
7895                 return rte_flow_error_set(error, ENOTSUP,
7896                                           RTE_FLOW_ERROR_TYPE_ACTION,
7897                                           NULL, "too many header modify"
7898                                           " actions to support");
7899         }
7900         /* Eswitch egress mirror and modify flow has limitation on CX5 */
7901         if (fdb_mirror_limit && modify_after_mirror)
7902                 return rte_flow_error_set(error, EINVAL,
7903                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7904                                 "sample before modify action is not supported");
7905         return 0;
7906 }
7907
7908 /**
7909  * Internal preparation function. Allocates the DV flow size,
7910  * this size is constant.
7911  *
7912  * @param[in] dev
7913  *   Pointer to the rte_eth_dev structure.
7914  * @param[in] attr
7915  *   Pointer to the flow attributes.
7916  * @param[in] items
7917  *   Pointer to the list of items.
7918  * @param[in] actions
7919  *   Pointer to the list of actions.
7920  * @param[out] error
7921  *   Pointer to the error structure.
7922  *
7923  * @return
7924  *   Pointer to mlx5_flow object on success,
7925  *   otherwise NULL and rte_errno is set.
7926  */
7927 static struct mlx5_flow *
7928 flow_dv_prepare(struct rte_eth_dev *dev,
7929                 const struct rte_flow_attr *attr __rte_unused,
7930                 const struct rte_flow_item items[] __rte_unused,
7931                 const struct rte_flow_action actions[] __rte_unused,
7932                 struct rte_flow_error *error)
7933 {
7934         uint32_t handle_idx = 0;
7935         struct mlx5_flow *dev_flow;
7936         struct mlx5_flow_handle *dev_handle;
7937         struct mlx5_priv *priv = dev->data->dev_private;
7938         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
7939
7940         MLX5_ASSERT(wks);
7941         wks->skip_matcher_reg = 0;
7942         wks->policy = NULL;
7943         wks->final_policy = NULL;
7944         /* In case of corrupting the memory. */
7945         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
7946                 rte_flow_error_set(error, ENOSPC,
7947                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7948                                    "not free temporary device flow");
7949                 return NULL;
7950         }
7951         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
7952                                    &handle_idx);
7953         if (!dev_handle) {
7954                 rte_flow_error_set(error, ENOMEM,
7955                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7956                                    "not enough memory to create flow handle");
7957                 return NULL;
7958         }
7959         MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows));
7960         dev_flow = &wks->flows[wks->flow_idx++];
7961         memset(dev_flow, 0, sizeof(*dev_flow));
7962         dev_flow->handle = dev_handle;
7963         dev_flow->handle_idx = handle_idx;
7964         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
7965         dev_flow->ingress = attr->ingress;
7966         dev_flow->dv.transfer = attr->transfer;
7967         return dev_flow;
7968 }
7969
7970 #ifdef RTE_LIBRTE_MLX5_DEBUG
7971 /**
7972  * Sanity check for match mask and value. Similar to check_valid_spec() in
7973  * kernel driver. If unmasked bit is present in value, it returns failure.
7974  *
7975  * @param match_mask
7976  *   pointer to match mask buffer.
7977  * @param match_value
7978  *   pointer to match value buffer.
7979  *
7980  * @return
7981  *   0 if valid, -EINVAL otherwise.
7982  */
7983 static int
7984 flow_dv_check_valid_spec(void *match_mask, void *match_value)
7985 {
7986         uint8_t *m = match_mask;
7987         uint8_t *v = match_value;
7988         unsigned int i;
7989
7990         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
7991                 if (v[i] & ~m[i]) {
7992                         DRV_LOG(ERR,
7993                                 "match_value differs from match_criteria"
7994                                 " %p[%u] != %p[%u]",
7995                                 match_value, i, match_mask, i);
7996                         return -EINVAL;
7997                 }
7998         }
7999         return 0;
8000 }
8001 #endif
8002
8003 /**
8004  * Add match of ip_version.
8005  *
8006  * @param[in] group
8007  *   Flow group.
8008  * @param[in] headers_v
8009  *   Values header pointer.
8010  * @param[in] headers_m
8011  *   Masks header pointer.
8012  * @param[in] ip_version
8013  *   The IP version to set.
8014  */
8015 static inline void
8016 flow_dv_set_match_ip_version(uint32_t group,
8017                              void *headers_v,
8018                              void *headers_m,
8019                              uint8_t ip_version)
8020 {
8021         if (group == 0)
8022                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
8023         else
8024                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
8025                          ip_version);
8026         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
8027         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
8028         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
8029 }
8030
8031 /**
8032  * Add Ethernet item to matcher and to the value.
8033  *
8034  * @param[in, out] matcher
8035  *   Flow matcher.
8036  * @param[in, out] key
8037  *   Flow matcher value.
8038  * @param[in] item
8039  *   Flow pattern to translate.
8040  * @param[in] inner
8041  *   Item is inner pattern.
8042  */
8043 static void
8044 flow_dv_translate_item_eth(void *matcher, void *key,
8045                            const struct rte_flow_item *item, int inner,
8046                            uint32_t group)
8047 {
8048         const struct rte_flow_item_eth *eth_m = item->mask;
8049         const struct rte_flow_item_eth *eth_v = item->spec;
8050         const struct rte_flow_item_eth nic_mask = {
8051                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8052                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8053                 .type = RTE_BE16(0xffff),
8054                 .has_vlan = 0,
8055         };
8056         void *hdrs_m;
8057         void *hdrs_v;
8058         char *l24_v;
8059         unsigned int i;
8060
8061         if (!eth_v)
8062                 return;
8063         if (!eth_m)
8064                 eth_m = &nic_mask;
8065         if (inner) {
8066                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8067                                          inner_headers);
8068                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8069         } else {
8070                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8071                                          outer_headers);
8072                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8073         }
8074         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
8075                &eth_m->dst, sizeof(eth_m->dst));
8076         /* The value must be in the range of the mask. */
8077         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
8078         for (i = 0; i < sizeof(eth_m->dst); ++i)
8079                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
8080         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
8081                &eth_m->src, sizeof(eth_m->src));
8082         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
8083         /* The value must be in the range of the mask. */
8084         for (i = 0; i < sizeof(eth_m->dst); ++i)
8085                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
8086         /*
8087          * HW supports match on one Ethertype, the Ethertype following the last
8088          * VLAN tag of the packet (see PRM).
8089          * Set match on ethertype only if ETH header is not followed by VLAN.
8090          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8091          * ethertype, and use ip_version field instead.
8092          * eCPRI over Ether layer will use type value 0xAEFE.
8093          */
8094         if (eth_m->type == 0xFFFF) {
8095                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
8096                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8097                 switch (eth_v->type) {
8098                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8099                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8100                         return;
8101                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
8102                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8103                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8104                         return;
8105                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8106                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8107                         return;
8108                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8109                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8110                         return;
8111                 default:
8112                         break;
8113                 }
8114         }
8115         if (eth_m->has_vlan) {
8116                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8117                 if (eth_v->has_vlan) {
8118                         /*
8119                          * Here, when also has_more_vlan field in VLAN item is
8120                          * not set, only single-tagged packets will be matched.
8121                          */
8122                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8123                         return;
8124                 }
8125         }
8126         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8127                  rte_be_to_cpu_16(eth_m->type));
8128         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
8129         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
8130 }
8131
8132 /**
8133  * Add VLAN item to matcher and to the value.
8134  *
8135  * @param[in, out] dev_flow
8136  *   Flow descriptor.
8137  * @param[in, out] matcher
8138  *   Flow matcher.
8139  * @param[in, out] key
8140  *   Flow matcher value.
8141  * @param[in] item
8142  *   Flow pattern to translate.
8143  * @param[in] inner
8144  *   Item is inner pattern.
8145  */
8146 static void
8147 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
8148                             void *matcher, void *key,
8149                             const struct rte_flow_item *item,
8150                             int inner, uint32_t group)
8151 {
8152         const struct rte_flow_item_vlan *vlan_m = item->mask;
8153         const struct rte_flow_item_vlan *vlan_v = item->spec;
8154         void *hdrs_m;
8155         void *hdrs_v;
8156         uint16_t tci_m;
8157         uint16_t tci_v;
8158
8159         if (inner) {
8160                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8161                                          inner_headers);
8162                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8163         } else {
8164                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8165                                          outer_headers);
8166                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8167                 /*
8168                  * This is workaround, masks are not supported,
8169                  * and pre-validated.
8170                  */
8171                 if (vlan_v)
8172                         dev_flow->handle->vf_vlan.tag =
8173                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
8174         }
8175         /*
8176          * When VLAN item exists in flow, mark packet as tagged,
8177          * even if TCI is not specified.
8178          */
8179         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
8180                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8181                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8182         }
8183         if (!vlan_v)
8184                 return;
8185         if (!vlan_m)
8186                 vlan_m = &rte_flow_item_vlan_mask;
8187         tci_m = rte_be_to_cpu_16(vlan_m->tci);
8188         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
8189         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
8190         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
8191         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
8192         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
8193         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
8194         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
8195         /*
8196          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8197          * ethertype, and use ip_version field instead.
8198          */
8199         if (vlan_m->inner_type == 0xFFFF) {
8200                 switch (vlan_v->inner_type) {
8201                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8202                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8203                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8204                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8205                         return;
8206                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8207                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8208                         return;
8209                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8210                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8211                         return;
8212                 default:
8213                         break;
8214                 }
8215         }
8216         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
8217                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8218                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8219                 /* Only one vlan_tag bit can be set. */
8220                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8221                 return;
8222         }
8223         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8224                  rte_be_to_cpu_16(vlan_m->inner_type));
8225         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
8226                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
8227 }
8228
8229 /**
8230  * Add IPV4 item to matcher and to the value.
8231  *
8232  * @param[in, out] matcher
8233  *   Flow matcher.
8234  * @param[in, out] key
8235  *   Flow matcher value.
8236  * @param[in] item
8237  *   Flow pattern to translate.
8238  * @param[in] inner
8239  *   Item is inner pattern.
8240  * @param[in] group
8241  *   The group to insert the rule.
8242  */
8243 static void
8244 flow_dv_translate_item_ipv4(void *matcher, void *key,
8245                             const struct rte_flow_item *item,
8246                             int inner, uint32_t group)
8247 {
8248         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
8249         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
8250         const struct rte_flow_item_ipv4 nic_mask = {
8251                 .hdr = {
8252                         .src_addr = RTE_BE32(0xffffffff),
8253                         .dst_addr = RTE_BE32(0xffffffff),
8254                         .type_of_service = 0xff,
8255                         .next_proto_id = 0xff,
8256                         .time_to_live = 0xff,
8257                 },
8258         };
8259         void *headers_m;
8260         void *headers_v;
8261         char *l24_m;
8262         char *l24_v;
8263         uint8_t tos;
8264
8265         if (inner) {
8266                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8267                                          inner_headers);
8268                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8269         } else {
8270                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8271                                          outer_headers);
8272                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8273         }
8274         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
8275         if (!ipv4_v)
8276                 return;
8277         if (!ipv4_m)
8278                 ipv4_m = &nic_mask;
8279         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8280                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8281         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8282                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8283         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
8284         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
8285         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8286                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8287         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8288                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8289         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
8290         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
8291         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
8292         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
8293                  ipv4_m->hdr.type_of_service);
8294         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
8295         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
8296                  ipv4_m->hdr.type_of_service >> 2);
8297         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
8298         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8299                  ipv4_m->hdr.next_proto_id);
8300         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8301                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
8302         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8303                  ipv4_m->hdr.time_to_live);
8304         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8305                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
8306         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8307                  !!(ipv4_m->hdr.fragment_offset));
8308         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8309                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
8310 }
8311
8312 /**
8313  * Add IPV6 item to matcher and to the value.
8314  *
8315  * @param[in, out] matcher
8316  *   Flow matcher.
8317  * @param[in, out] key
8318  *   Flow matcher value.
8319  * @param[in] item
8320  *   Flow pattern to translate.
8321  * @param[in] inner
8322  *   Item is inner pattern.
8323  * @param[in] group
8324  *   The group to insert the rule.
8325  */
8326 static void
8327 flow_dv_translate_item_ipv6(void *matcher, void *key,
8328                             const struct rte_flow_item *item,
8329                             int inner, uint32_t group)
8330 {
8331         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
8332         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
8333         const struct rte_flow_item_ipv6 nic_mask = {
8334                 .hdr = {
8335                         .src_addr =
8336                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8337                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8338                         .dst_addr =
8339                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8340                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8341                         .vtc_flow = RTE_BE32(0xffffffff),
8342                         .proto = 0xff,
8343                         .hop_limits = 0xff,
8344                 },
8345         };
8346         void *headers_m;
8347         void *headers_v;
8348         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8349         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8350         char *l24_m;
8351         char *l24_v;
8352         uint32_t vtc_m;
8353         uint32_t vtc_v;
8354         int i;
8355         int size;
8356
8357         if (inner) {
8358                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8359                                          inner_headers);
8360                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8361         } else {
8362                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8363                                          outer_headers);
8364                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8365         }
8366         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
8367         if (!ipv6_v)
8368                 return;
8369         if (!ipv6_m)
8370                 ipv6_m = &nic_mask;
8371         size = sizeof(ipv6_m->hdr.dst_addr);
8372         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8373                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8374         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8375                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8376         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
8377         for (i = 0; i < size; ++i)
8378                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
8379         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8380                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8381         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8382                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8383         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
8384         for (i = 0; i < size; ++i)
8385                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
8386         /* TOS. */
8387         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
8388         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
8389         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
8390         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
8391         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
8392         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
8393         /* Label. */
8394         if (inner) {
8395                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
8396                          vtc_m);
8397                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
8398                          vtc_v);
8399         } else {
8400                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
8401                          vtc_m);
8402                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
8403                          vtc_v);
8404         }
8405         /* Protocol. */
8406         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8407                  ipv6_m->hdr.proto);
8408         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8409                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
8410         /* Hop limit. */
8411         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8412                  ipv6_m->hdr.hop_limits);
8413         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8414                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
8415         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8416                  !!(ipv6_m->has_frag_ext));
8417         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8418                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
8419 }
8420
8421 /**
8422  * Add IPV6 fragment extension item to matcher and to the value.
8423  *
8424  * @param[in, out] matcher
8425  *   Flow matcher.
8426  * @param[in, out] key
8427  *   Flow matcher value.
8428  * @param[in] item
8429  *   Flow pattern to translate.
8430  * @param[in] inner
8431  *   Item is inner pattern.
8432  */
8433 static void
8434 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
8435                                      const struct rte_flow_item *item,
8436                                      int inner)
8437 {
8438         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
8439         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
8440         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
8441                 .hdr = {
8442                         .next_header = 0xff,
8443                         .frag_data = RTE_BE16(0xffff),
8444                 },
8445         };
8446         void *headers_m;
8447         void *headers_v;
8448
8449         if (inner) {
8450                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8451                                          inner_headers);
8452                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8453         } else {
8454                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8455                                          outer_headers);
8456                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8457         }
8458         /* IPv6 fragment extension item exists, so packet is IP fragment. */
8459         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
8460         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
8461         if (!ipv6_frag_ext_v)
8462                 return;
8463         if (!ipv6_frag_ext_m)
8464                 ipv6_frag_ext_m = &nic_mask;
8465         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8466                  ipv6_frag_ext_m->hdr.next_header);
8467         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8468                  ipv6_frag_ext_v->hdr.next_header &
8469                  ipv6_frag_ext_m->hdr.next_header);
8470 }
8471
8472 /**
8473  * Add TCP item to matcher and to the value.
8474  *
8475  * @param[in, out] matcher
8476  *   Flow matcher.
8477  * @param[in, out] key
8478  *   Flow matcher value.
8479  * @param[in] item
8480  *   Flow pattern to translate.
8481  * @param[in] inner
8482  *   Item is inner pattern.
8483  */
8484 static void
8485 flow_dv_translate_item_tcp(void *matcher, void *key,
8486                            const struct rte_flow_item *item,
8487                            int inner)
8488 {
8489         const struct rte_flow_item_tcp *tcp_m = item->mask;
8490         const struct rte_flow_item_tcp *tcp_v = item->spec;
8491         void *headers_m;
8492         void *headers_v;
8493
8494         if (inner) {
8495                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8496                                          inner_headers);
8497                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8498         } else {
8499                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8500                                          outer_headers);
8501                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8502         }
8503         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8504         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
8505         if (!tcp_v)
8506                 return;
8507         if (!tcp_m)
8508                 tcp_m = &rte_flow_item_tcp_mask;
8509         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
8510                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
8511         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
8512                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
8513         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
8514                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
8515         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
8516                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
8517         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
8518                  tcp_m->hdr.tcp_flags);
8519         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
8520                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
8521 }
8522
8523 /**
8524  * Add UDP item to matcher and to the value.
8525  *
8526  * @param[in, out] matcher
8527  *   Flow matcher.
8528  * @param[in, out] key
8529  *   Flow matcher value.
8530  * @param[in] item
8531  *   Flow pattern to translate.
8532  * @param[in] inner
8533  *   Item is inner pattern.
8534  */
8535 static void
8536 flow_dv_translate_item_udp(void *matcher, void *key,
8537                            const struct rte_flow_item *item,
8538                            int inner)
8539 {
8540         const struct rte_flow_item_udp *udp_m = item->mask;
8541         const struct rte_flow_item_udp *udp_v = item->spec;
8542         void *headers_m;
8543         void *headers_v;
8544
8545         if (inner) {
8546                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8547                                          inner_headers);
8548                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8549         } else {
8550                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8551                                          outer_headers);
8552                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8553         }
8554         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8555         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
8556         if (!udp_v)
8557                 return;
8558         if (!udp_m)
8559                 udp_m = &rte_flow_item_udp_mask;
8560         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
8561                  rte_be_to_cpu_16(udp_m->hdr.src_port));
8562         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
8563                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
8564         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
8565                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
8566         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
8567                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
8568 }
8569
8570 /**
8571  * Add GRE optional Key item to matcher and to the value.
8572  *
8573  * @param[in, out] matcher
8574  *   Flow matcher.
8575  * @param[in, out] key
8576  *   Flow matcher value.
8577  * @param[in] item
8578  *   Flow pattern to translate.
8579  * @param[in] inner
8580  *   Item is inner pattern.
8581  */
8582 static void
8583 flow_dv_translate_item_gre_key(void *matcher, void *key,
8584                                    const struct rte_flow_item *item)
8585 {
8586         const rte_be32_t *key_m = item->mask;
8587         const rte_be32_t *key_v = item->spec;
8588         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8589         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8590         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
8591
8592         /* GRE K bit must be on and should already be validated */
8593         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
8594         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
8595         if (!key_v)
8596                 return;
8597         if (!key_m)
8598                 key_m = &gre_key_default_mask;
8599         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
8600                  rte_be_to_cpu_32(*key_m) >> 8);
8601         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
8602                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
8603         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
8604                  rte_be_to_cpu_32(*key_m) & 0xFF);
8605         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
8606                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
8607 }
8608
8609 /**
8610  * Add GRE item to matcher and to the value.
8611  *
8612  * @param[in, out] matcher
8613  *   Flow matcher.
8614  * @param[in, out] key
8615  *   Flow matcher value.
8616  * @param[in] item
8617  *   Flow pattern to translate.
8618  * @param[in] inner
8619  *   Item is inner pattern.
8620  */
8621 static void
8622 flow_dv_translate_item_gre(void *matcher, void *key,
8623                            const struct rte_flow_item *item,
8624                            int inner)
8625 {
8626         const struct rte_flow_item_gre *gre_m = item->mask;
8627         const struct rte_flow_item_gre *gre_v = item->spec;
8628         void *headers_m;
8629         void *headers_v;
8630         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8631         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8632         struct {
8633                 union {
8634                         __extension__
8635                         struct {
8636                                 uint16_t version:3;
8637                                 uint16_t rsvd0:9;
8638                                 uint16_t s_present:1;
8639                                 uint16_t k_present:1;
8640                                 uint16_t rsvd_bit1:1;
8641                                 uint16_t c_present:1;
8642                         };
8643                         uint16_t value;
8644                 };
8645         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
8646
8647         if (inner) {
8648                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8649                                          inner_headers);
8650                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8651         } else {
8652                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8653                                          outer_headers);
8654                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8655         }
8656         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8657         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
8658         if (!gre_v)
8659                 return;
8660         if (!gre_m)
8661                 gre_m = &rte_flow_item_gre_mask;
8662         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
8663                  rte_be_to_cpu_16(gre_m->protocol));
8664         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
8665                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
8666         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
8667         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
8668         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
8669                  gre_crks_rsvd0_ver_m.c_present);
8670         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
8671                  gre_crks_rsvd0_ver_v.c_present &
8672                  gre_crks_rsvd0_ver_m.c_present);
8673         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
8674                  gre_crks_rsvd0_ver_m.k_present);
8675         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
8676                  gre_crks_rsvd0_ver_v.k_present &
8677                  gre_crks_rsvd0_ver_m.k_present);
8678         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
8679                  gre_crks_rsvd0_ver_m.s_present);
8680         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
8681                  gre_crks_rsvd0_ver_v.s_present &
8682                  gre_crks_rsvd0_ver_m.s_present);
8683 }
8684
8685 /**
8686  * Add NVGRE item to matcher and to the value.
8687  *
8688  * @param[in, out] matcher
8689  *   Flow matcher.
8690  * @param[in, out] key
8691  *   Flow matcher value.
8692  * @param[in] item
8693  *   Flow pattern to translate.
8694  * @param[in] inner
8695  *   Item is inner pattern.
8696  */
8697 static void
8698 flow_dv_translate_item_nvgre(void *matcher, void *key,
8699                              const struct rte_flow_item *item,
8700                              int inner)
8701 {
8702         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
8703         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
8704         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8705         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8706         const char *tni_flow_id_m;
8707         const char *tni_flow_id_v;
8708         char *gre_key_m;
8709         char *gre_key_v;
8710         int size;
8711         int i;
8712
8713         /* For NVGRE, GRE header fields must be set with defined values. */
8714         const struct rte_flow_item_gre gre_spec = {
8715                 .c_rsvd0_ver = RTE_BE16(0x2000),
8716                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
8717         };
8718         const struct rte_flow_item_gre gre_mask = {
8719                 .c_rsvd0_ver = RTE_BE16(0xB000),
8720                 .protocol = RTE_BE16(UINT16_MAX),
8721         };
8722         const struct rte_flow_item gre_item = {
8723                 .spec = &gre_spec,
8724                 .mask = &gre_mask,
8725                 .last = NULL,
8726         };
8727         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
8728         if (!nvgre_v)
8729                 return;
8730         if (!nvgre_m)
8731                 nvgre_m = &rte_flow_item_nvgre_mask;
8732         tni_flow_id_m = (const char *)nvgre_m->tni;
8733         tni_flow_id_v = (const char *)nvgre_v->tni;
8734         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
8735         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
8736         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
8737         memcpy(gre_key_m, tni_flow_id_m, size);
8738         for (i = 0; i < size; ++i)
8739                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
8740 }
8741
8742 /**
8743  * Add VXLAN item to matcher and to the value.
8744  *
8745  * @param[in] dev
8746  *   Pointer to the Ethernet device structure.
8747  * @param[in] attr
8748  *   Flow rule attributes.
8749  * @param[in, out] matcher
8750  *   Flow matcher.
8751  * @param[in, out] key
8752  *   Flow matcher value.
8753  * @param[in] item
8754  *   Flow pattern to translate.
8755  * @param[in] inner
8756  *   Item is inner pattern.
8757  */
8758 static void
8759 flow_dv_translate_item_vxlan(struct rte_eth_dev *dev,
8760                              const struct rte_flow_attr *attr,
8761                              void *matcher, void *key,
8762                              const struct rte_flow_item *item,
8763                              int inner)
8764 {
8765         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
8766         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
8767         void *headers_m;
8768         void *headers_v;
8769         void *misc5_m;
8770         void *misc5_v;
8771         uint32_t *tunnel_header_v;
8772         uint32_t *tunnel_header_m;
8773         uint16_t dport;
8774         struct mlx5_priv *priv = dev->data->dev_private;
8775         const struct rte_flow_item_vxlan nic_mask = {
8776                 .vni = "\xff\xff\xff",
8777                 .rsvd1 = 0xff,
8778         };
8779
8780         if (inner) {
8781                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8782                                          inner_headers);
8783                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8784         } else {
8785                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8786                                          outer_headers);
8787                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8788         }
8789         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8790                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8791         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8792                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8793                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8794         }
8795         if (!vxlan_v)
8796                 return;
8797         if (!vxlan_m) {
8798                 if ((!attr->group && !priv->sh->tunnel_header_0_1) ||
8799                     (attr->group && !priv->sh->misc5_cap))
8800                         vxlan_m = &rte_flow_item_vxlan_mask;
8801                 else
8802                         vxlan_m = &nic_mask;
8803         }
8804         if ((!attr->group && !attr->transfer && !priv->sh->tunnel_header_0_1) ||
8805             ((attr->group || attr->transfer) && !priv->sh->misc5_cap)) {
8806                 void *misc_m;
8807                 void *misc_v;
8808                 char *vni_m;
8809                 char *vni_v;
8810                 int size;
8811                 int i;
8812                 misc_m = MLX5_ADDR_OF(fte_match_param,
8813                                       matcher, misc_parameters);
8814                 misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8815                 size = sizeof(vxlan_m->vni);
8816                 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
8817                 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
8818                 memcpy(vni_m, vxlan_m->vni, size);
8819                 for (i = 0; i < size; ++i)
8820                         vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8821                 return;
8822         }
8823         misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
8824         misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
8825         tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8826                                                    misc5_v,
8827                                                    tunnel_header_1);
8828         tunnel_header_m = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8829                                                    misc5_m,
8830                                                    tunnel_header_1);
8831         *tunnel_header_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) |
8832                            (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 |
8833                            (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16;
8834         if (*tunnel_header_v)
8835                 *tunnel_header_m = vxlan_m->vni[0] |
8836                         vxlan_m->vni[1] << 8 |
8837                         vxlan_m->vni[2] << 16;
8838         else
8839                 *tunnel_header_m = 0x0;
8840         *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24;
8841         if (vxlan_v->rsvd1 & vxlan_m->rsvd1)
8842                 *tunnel_header_m |= vxlan_m->rsvd1 << 24;
8843 }
8844
8845 /**
8846  * Add VXLAN-GPE item to matcher and to the value.
8847  *
8848  * @param[in, out] matcher
8849  *   Flow matcher.
8850  * @param[in, out] key
8851  *   Flow matcher value.
8852  * @param[in] item
8853  *   Flow pattern to translate.
8854  * @param[in] inner
8855  *   Item is inner pattern.
8856  */
8857
8858 static void
8859 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
8860                                  const struct rte_flow_item *item, int inner)
8861 {
8862         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
8863         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
8864         void *headers_m;
8865         void *headers_v;
8866         void *misc_m =
8867                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
8868         void *misc_v =
8869                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
8870         char *vni_m;
8871         char *vni_v;
8872         uint16_t dport;
8873         int size;
8874         int i;
8875         uint8_t flags_m = 0xff;
8876         uint8_t flags_v = 0xc;
8877
8878         if (inner) {
8879                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8880                                          inner_headers);
8881                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8882         } else {
8883                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8884                                          outer_headers);
8885                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8886         }
8887         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8888                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8889         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8890                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8891                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8892         }
8893         if (!vxlan_v)
8894                 return;
8895         if (!vxlan_m)
8896                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
8897         size = sizeof(vxlan_m->vni);
8898         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
8899         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
8900         memcpy(vni_m, vxlan_m->vni, size);
8901         for (i = 0; i < size; ++i)
8902                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8903         if (vxlan_m->flags) {
8904                 flags_m = vxlan_m->flags;
8905                 flags_v = vxlan_v->flags;
8906         }
8907         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
8908         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
8909         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
8910                  vxlan_m->protocol);
8911         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
8912                  vxlan_v->protocol);
8913 }
8914
8915 /**
8916  * Add Geneve item to matcher and to the value.
8917  *
8918  * @param[in, out] matcher
8919  *   Flow matcher.
8920  * @param[in, out] key
8921  *   Flow matcher value.
8922  * @param[in] item
8923  *   Flow pattern to translate.
8924  * @param[in] inner
8925  *   Item is inner pattern.
8926  */
8927
8928 static void
8929 flow_dv_translate_item_geneve(void *matcher, void *key,
8930                               const struct rte_flow_item *item, int inner)
8931 {
8932         const struct rte_flow_item_geneve *geneve_m = item->mask;
8933         const struct rte_flow_item_geneve *geneve_v = item->spec;
8934         void *headers_m;
8935         void *headers_v;
8936         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8937         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8938         uint16_t dport;
8939         uint16_t gbhdr_m;
8940         uint16_t gbhdr_v;
8941         char *vni_m;
8942         char *vni_v;
8943         size_t size, i;
8944
8945         if (inner) {
8946                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8947                                          inner_headers);
8948                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8949         } else {
8950                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8951                                          outer_headers);
8952                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8953         }
8954         dport = MLX5_UDP_PORT_GENEVE;
8955         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8956                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8957                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8958         }
8959         if (!geneve_v)
8960                 return;
8961         if (!geneve_m)
8962                 geneve_m = &rte_flow_item_geneve_mask;
8963         size = sizeof(geneve_m->vni);
8964         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
8965         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
8966         memcpy(vni_m, geneve_m->vni, size);
8967         for (i = 0; i < size; ++i)
8968                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
8969         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
8970                  rte_be_to_cpu_16(geneve_m->protocol));
8971         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
8972                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
8973         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
8974         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
8975         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
8976                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
8977         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
8978                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
8979         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
8980                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
8981         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
8982                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
8983                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
8984 }
8985
8986 /**
8987  * Create Geneve TLV option resource.
8988  *
8989  * @param dev[in, out]
8990  *   Pointer to rte_eth_dev structure.
8991  * @param[in, out] tag_be24
8992  *   Tag value in big endian then R-shift 8.
8993  * @parm[in, out] dev_flow
8994  *   Pointer to the dev_flow.
8995  * @param[out] error
8996  *   pointer to error structure.
8997  *
8998  * @return
8999  *   0 on success otherwise -errno and errno is set.
9000  */
9001
9002 int
9003 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
9004                                              const struct rte_flow_item *item,
9005                                              struct rte_flow_error *error)
9006 {
9007         struct mlx5_priv *priv = dev->data->dev_private;
9008         struct mlx5_dev_ctx_shared *sh = priv->sh;
9009         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
9010                         sh->geneve_tlv_option_resource;
9011         struct mlx5_devx_obj *obj;
9012         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9013         int ret = 0;
9014
9015         if (!geneve_opt_v)
9016                 return -1;
9017         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
9018         if (geneve_opt_resource != NULL) {
9019                 if (geneve_opt_resource->option_class ==
9020                         geneve_opt_v->option_class &&
9021                         geneve_opt_resource->option_type ==
9022                         geneve_opt_v->option_type &&
9023                         geneve_opt_resource->length ==
9024                         geneve_opt_v->option_len) {
9025                         /* We already have GENVE TLV option obj allocated. */
9026                         __atomic_fetch_add(&geneve_opt_resource->refcnt, 1,
9027                                            __ATOMIC_RELAXED);
9028                 } else {
9029                         ret = rte_flow_error_set(error, ENOMEM,
9030                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9031                                 "Only one GENEVE TLV option supported");
9032                         goto exit;
9033                 }
9034         } else {
9035                 /* Create a GENEVE TLV object and resource. */
9036                 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->ctx,
9037                                 geneve_opt_v->option_class,
9038                                 geneve_opt_v->option_type,
9039                                 geneve_opt_v->option_len);
9040                 if (!obj) {
9041                         ret = rte_flow_error_set(error, ENODATA,
9042                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9043                                 "Failed to create GENEVE TLV Devx object");
9044                         goto exit;
9045                 }
9046                 sh->geneve_tlv_option_resource =
9047                                 mlx5_malloc(MLX5_MEM_ZERO,
9048                                                 sizeof(*geneve_opt_resource),
9049                                                 0, SOCKET_ID_ANY);
9050                 if (!sh->geneve_tlv_option_resource) {
9051                         claim_zero(mlx5_devx_cmd_destroy(obj));
9052                         ret = rte_flow_error_set(error, ENOMEM,
9053                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9054                                 "GENEVE TLV object memory allocation failed");
9055                         goto exit;
9056                 }
9057                 geneve_opt_resource = sh->geneve_tlv_option_resource;
9058                 geneve_opt_resource->obj = obj;
9059                 geneve_opt_resource->option_class = geneve_opt_v->option_class;
9060                 geneve_opt_resource->option_type = geneve_opt_v->option_type;
9061                 geneve_opt_resource->length = geneve_opt_v->option_len;
9062                 __atomic_store_n(&geneve_opt_resource->refcnt, 1,
9063                                 __ATOMIC_RELAXED);
9064         }
9065 exit:
9066         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
9067         return ret;
9068 }
9069
9070 /**
9071  * Add Geneve TLV option item to matcher.
9072  *
9073  * @param[in, out] dev
9074  *   Pointer to rte_eth_dev structure.
9075  * @param[in, out] matcher
9076  *   Flow matcher.
9077  * @param[in, out] key
9078  *   Flow matcher value.
9079  * @param[in] item
9080  *   Flow pattern to translate.
9081  * @param[out] error
9082  *   Pointer to error structure.
9083  */
9084 static int
9085 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher,
9086                                   void *key, const struct rte_flow_item *item,
9087                                   struct rte_flow_error *error)
9088 {
9089         const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask;
9090         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9091         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9092         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9093         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9094                         misc_parameters_3);
9095         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9096         rte_be32_t opt_data_key = 0, opt_data_mask = 0;
9097         int ret = 0;
9098
9099         if (!geneve_opt_v)
9100                 return -1;
9101         if (!geneve_opt_m)
9102                 geneve_opt_m = &rte_flow_item_geneve_opt_mask;
9103         ret = flow_dev_geneve_tlv_option_resource_register(dev, item,
9104                                                            error);
9105         if (ret) {
9106                 DRV_LOG(ERR, "Failed to create geneve_tlv_obj");
9107                 return ret;
9108         }
9109         /*
9110          * Set the option length in GENEVE header if not requested.
9111          * The GENEVE TLV option length is expressed by the option length field
9112          * in the GENEVE header.
9113          * If the option length was not requested but the GENEVE TLV option item
9114          * is present we set the option length field implicitly.
9115          */
9116         if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) {
9117                 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9118                          MLX5_GENEVE_OPTLEN_MASK);
9119                 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9120                          geneve_opt_v->option_len + 1);
9121         }
9122         /* Set the data. */
9123         if (geneve_opt_v->data) {
9124                 memcpy(&opt_data_key, geneve_opt_v->data,
9125                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9126                                 sizeof(opt_data_key)));
9127                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9128                                 sizeof(opt_data_key));
9129                 memcpy(&opt_data_mask, geneve_opt_m->data,
9130                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9131                                 sizeof(opt_data_mask)));
9132                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9133                                 sizeof(opt_data_mask));
9134                 MLX5_SET(fte_match_set_misc3, misc3_m,
9135                                 geneve_tlv_option_0_data,
9136                                 rte_be_to_cpu_32(opt_data_mask));
9137                 MLX5_SET(fte_match_set_misc3, misc3_v,
9138                                 geneve_tlv_option_0_data,
9139                         rte_be_to_cpu_32(opt_data_key & opt_data_mask));
9140         }
9141         return ret;
9142 }
9143
9144 /**
9145  * Add MPLS item to matcher and to the value.
9146  *
9147  * @param[in, out] matcher
9148  *   Flow matcher.
9149  * @param[in, out] key
9150  *   Flow matcher value.
9151  * @param[in] item
9152  *   Flow pattern to translate.
9153  * @param[in] prev_layer
9154  *   The protocol layer indicated in previous item.
9155  * @param[in] inner
9156  *   Item is inner pattern.
9157  */
9158 static void
9159 flow_dv_translate_item_mpls(void *matcher, void *key,
9160                             const struct rte_flow_item *item,
9161                             uint64_t prev_layer,
9162                             int inner)
9163 {
9164         const uint32_t *in_mpls_m = item->mask;
9165         const uint32_t *in_mpls_v = item->spec;
9166         uint32_t *out_mpls_m = 0;
9167         uint32_t *out_mpls_v = 0;
9168         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9169         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9170         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
9171                                      misc_parameters_2);
9172         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9173         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9174         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9175
9176         switch (prev_layer) {
9177         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9178                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
9179                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9180                          MLX5_UDP_PORT_MPLS);
9181                 break;
9182         case MLX5_FLOW_LAYER_GRE:
9183                 /* Fall-through. */
9184         case MLX5_FLOW_LAYER_GRE_KEY:
9185                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
9186                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
9187                          RTE_ETHER_TYPE_MPLS);
9188                 break;
9189         default:
9190                 break;
9191         }
9192         if (!in_mpls_v)
9193                 return;
9194         if (!in_mpls_m)
9195                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
9196         switch (prev_layer) {
9197         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9198                 out_mpls_m =
9199                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9200                                                  outer_first_mpls_over_udp);
9201                 out_mpls_v =
9202                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9203                                                  outer_first_mpls_over_udp);
9204                 break;
9205         case MLX5_FLOW_LAYER_GRE:
9206                 out_mpls_m =
9207                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9208                                                  outer_first_mpls_over_gre);
9209                 out_mpls_v =
9210                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9211                                                  outer_first_mpls_over_gre);
9212                 break;
9213         default:
9214                 /* Inner MPLS not over GRE is not supported. */
9215                 if (!inner) {
9216                         out_mpls_m =
9217                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9218                                                          misc2_m,
9219                                                          outer_first_mpls);
9220                         out_mpls_v =
9221                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9222                                                          misc2_v,
9223                                                          outer_first_mpls);
9224                 }
9225                 break;
9226         }
9227         if (out_mpls_m && out_mpls_v) {
9228                 *out_mpls_m = *in_mpls_m;
9229                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
9230         }
9231 }
9232
9233 /**
9234  * Add metadata register item to matcher
9235  *
9236  * @param[in, out] matcher
9237  *   Flow matcher.
9238  * @param[in, out] key
9239  *   Flow matcher value.
9240  * @param[in] reg_type
9241  *   Type of device metadata register
9242  * @param[in] value
9243  *   Register value
9244  * @param[in] mask
9245  *   Register mask
9246  */
9247 static void
9248 flow_dv_match_meta_reg(void *matcher, void *key,
9249                        enum modify_reg reg_type,
9250                        uint32_t data, uint32_t mask)
9251 {
9252         void *misc2_m =
9253                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
9254         void *misc2_v =
9255                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9256         uint32_t temp;
9257
9258         data &= mask;
9259         switch (reg_type) {
9260         case REG_A:
9261                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
9262                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
9263                 break;
9264         case REG_B:
9265                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
9266                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
9267                 break;
9268         case REG_C_0:
9269                 /*
9270                  * The metadata register C0 field might be divided into
9271                  * source vport index and META item value, we should set
9272                  * this field according to specified mask, not as whole one.
9273                  */
9274                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
9275                 temp |= mask;
9276                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
9277                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
9278                 temp &= ~mask;
9279                 temp |= data;
9280                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
9281                 break;
9282         case REG_C_1:
9283                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
9284                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
9285                 break;
9286         case REG_C_2:
9287                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
9288                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
9289                 break;
9290         case REG_C_3:
9291                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
9292                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
9293                 break;
9294         case REG_C_4:
9295                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
9296                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
9297                 break;
9298         case REG_C_5:
9299                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
9300                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
9301                 break;
9302         case REG_C_6:
9303                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
9304                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
9305                 break;
9306         case REG_C_7:
9307                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
9308                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
9309                 break;
9310         default:
9311                 MLX5_ASSERT(false);
9312                 break;
9313         }
9314 }
9315
9316 /**
9317  * Add MARK item to matcher
9318  *
9319  * @param[in] dev
9320  *   The device to configure through.
9321  * @param[in, out] matcher
9322  *   Flow matcher.
9323  * @param[in, out] key
9324  *   Flow matcher value.
9325  * @param[in] item
9326  *   Flow pattern to translate.
9327  */
9328 static void
9329 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
9330                             void *matcher, void *key,
9331                             const struct rte_flow_item *item)
9332 {
9333         struct mlx5_priv *priv = dev->data->dev_private;
9334         const struct rte_flow_item_mark *mark;
9335         uint32_t value;
9336         uint32_t mask;
9337
9338         mark = item->mask ? (const void *)item->mask :
9339                             &rte_flow_item_mark_mask;
9340         mask = mark->id & priv->sh->dv_mark_mask;
9341         mark = (const void *)item->spec;
9342         MLX5_ASSERT(mark);
9343         value = mark->id & priv->sh->dv_mark_mask & mask;
9344         if (mask) {
9345                 enum modify_reg reg;
9346
9347                 /* Get the metadata register index for the mark. */
9348                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
9349                 MLX5_ASSERT(reg > 0);
9350                 if (reg == REG_C_0) {
9351                         struct mlx5_priv *priv = dev->data->dev_private;
9352                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9353                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9354
9355                         mask &= msk_c0;
9356                         mask <<= shl_c0;
9357                         value <<= shl_c0;
9358                 }
9359                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9360         }
9361 }
9362
9363 /**
9364  * Add META item to matcher
9365  *
9366  * @param[in] dev
9367  *   The devich to configure through.
9368  * @param[in, out] matcher
9369  *   Flow matcher.
9370  * @param[in, out] key
9371  *   Flow matcher value.
9372  * @param[in] attr
9373  *   Attributes of flow that includes this item.
9374  * @param[in] item
9375  *   Flow pattern to translate.
9376  */
9377 static void
9378 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
9379                             void *matcher, void *key,
9380                             const struct rte_flow_attr *attr,
9381                             const struct rte_flow_item *item)
9382 {
9383         const struct rte_flow_item_meta *meta_m;
9384         const struct rte_flow_item_meta *meta_v;
9385
9386         meta_m = (const void *)item->mask;
9387         if (!meta_m)
9388                 meta_m = &rte_flow_item_meta_mask;
9389         meta_v = (const void *)item->spec;
9390         if (meta_v) {
9391                 int reg;
9392                 uint32_t value = meta_v->data;
9393                 uint32_t mask = meta_m->data;
9394
9395                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
9396                 if (reg < 0)
9397                         return;
9398                 MLX5_ASSERT(reg != REG_NON);
9399                 if (reg == REG_C_0) {
9400                         struct mlx5_priv *priv = dev->data->dev_private;
9401                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9402                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9403
9404                         mask &= msk_c0;
9405                         mask <<= shl_c0;
9406                         value <<= shl_c0;
9407                 }
9408                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9409         }
9410 }
9411
9412 /**
9413  * Add vport metadata Reg C0 item to matcher
9414  *
9415  * @param[in, out] matcher
9416  *   Flow matcher.
9417  * @param[in, out] key
9418  *   Flow matcher value.
9419  * @param[in] reg
9420  *   Flow pattern to translate.
9421  */
9422 static void
9423 flow_dv_translate_item_meta_vport(void *matcher, void *key,
9424                                   uint32_t value, uint32_t mask)
9425 {
9426         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
9427 }
9428
9429 /**
9430  * Add tag item to matcher
9431  *
9432  * @param[in] dev
9433  *   The devich to configure through.
9434  * @param[in, out] matcher
9435  *   Flow matcher.
9436  * @param[in, out] key
9437  *   Flow matcher value.
9438  * @param[in] item
9439  *   Flow pattern to translate.
9440  */
9441 static void
9442 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
9443                                 void *matcher, void *key,
9444                                 const struct rte_flow_item *item)
9445 {
9446         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
9447         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
9448         uint32_t mask, value;
9449
9450         MLX5_ASSERT(tag_v);
9451         value = tag_v->data;
9452         mask = tag_m ? tag_m->data : UINT32_MAX;
9453         if (tag_v->id == REG_C_0) {
9454                 struct mlx5_priv *priv = dev->data->dev_private;
9455                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9456                 uint32_t shl_c0 = rte_bsf32(msk_c0);
9457
9458                 mask &= msk_c0;
9459                 mask <<= shl_c0;
9460                 value <<= shl_c0;
9461         }
9462         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
9463 }
9464
9465 /**
9466  * Add TAG item to matcher
9467  *
9468  * @param[in] dev
9469  *   The devich to configure through.
9470  * @param[in, out] matcher
9471  *   Flow matcher.
9472  * @param[in, out] key
9473  *   Flow matcher value.
9474  * @param[in] item
9475  *   Flow pattern to translate.
9476  */
9477 static void
9478 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
9479                            void *matcher, void *key,
9480                            const struct rte_flow_item *item)
9481 {
9482         const struct rte_flow_item_tag *tag_v = item->spec;
9483         const struct rte_flow_item_tag *tag_m = item->mask;
9484         enum modify_reg reg;
9485
9486         MLX5_ASSERT(tag_v);
9487         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
9488         /* Get the metadata register index for the tag. */
9489         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
9490         MLX5_ASSERT(reg > 0);
9491         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
9492 }
9493
9494 /**
9495  * Add source vport match to the specified matcher.
9496  *
9497  * @param[in, out] matcher
9498  *   Flow matcher.
9499  * @param[in, out] key
9500  *   Flow matcher value.
9501  * @param[in] port
9502  *   Source vport value to match
9503  * @param[in] mask
9504  *   Mask
9505  */
9506 static void
9507 flow_dv_translate_item_source_vport(void *matcher, void *key,
9508                                     int16_t port, uint16_t mask)
9509 {
9510         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9511         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9512
9513         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
9514         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
9515 }
9516
9517 /**
9518  * Translate port-id item to eswitch match on  port-id.
9519  *
9520  * @param[in] dev
9521  *   The devich to configure through.
9522  * @param[in, out] matcher
9523  *   Flow matcher.
9524  * @param[in, out] key
9525  *   Flow matcher value.
9526  * @param[in] item
9527  *   Flow pattern to translate.
9528  * @param[in]
9529  *   Flow attributes.
9530  *
9531  * @return
9532  *   0 on success, a negative errno value otherwise.
9533  */
9534 static int
9535 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
9536                                void *key, const struct rte_flow_item *item,
9537                                const struct rte_flow_attr *attr)
9538 {
9539         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
9540         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
9541         struct mlx5_priv *priv;
9542         uint16_t mask, id;
9543
9544         mask = pid_m ? pid_m->id : 0xffff;
9545         id = pid_v ? pid_v->id : dev->data->port_id;
9546         priv = mlx5_port_to_eswitch_info(id, item == NULL);
9547         if (!priv)
9548                 return -rte_errno;
9549         /*
9550          * Translate to vport field or to metadata, depending on mode.
9551          * Kernel can use either misc.source_port or half of C0 metadata
9552          * register.
9553          */
9554         if (priv->vport_meta_mask) {
9555                 /*
9556                  * Provide the hint for SW steering library
9557                  * to insert the flow into ingress domain and
9558                  * save the extra vport match.
9559                  */
9560                 if (mask == 0xffff && priv->vport_id == 0xffff &&
9561                     priv->pf_bond < 0 && attr->transfer)
9562                         flow_dv_translate_item_source_vport
9563                                 (matcher, key, priv->vport_id, mask);
9564                 /*
9565                  * We should always set the vport metadata register,
9566                  * otherwise the SW steering library can drop
9567                  * the rule if wire vport metadata value is not zero,
9568                  * it depends on kernel configuration.
9569                  */
9570                 flow_dv_translate_item_meta_vport(matcher, key,
9571                                                   priv->vport_meta_tag,
9572                                                   priv->vport_meta_mask);
9573         } else {
9574                 flow_dv_translate_item_source_vport(matcher, key,
9575                                                     priv->vport_id, mask);
9576         }
9577         return 0;
9578 }
9579
9580 /**
9581  * Add ICMP6 item to matcher and to the value.
9582  *
9583  * @param[in, out] matcher
9584  *   Flow matcher.
9585  * @param[in, out] key
9586  *   Flow matcher value.
9587  * @param[in] item
9588  *   Flow pattern to translate.
9589  * @param[in] inner
9590  *   Item is inner pattern.
9591  */
9592 static void
9593 flow_dv_translate_item_icmp6(void *matcher, void *key,
9594                               const struct rte_flow_item *item,
9595                               int inner)
9596 {
9597         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
9598         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
9599         void *headers_m;
9600         void *headers_v;
9601         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9602                                      misc_parameters_3);
9603         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9604         if (inner) {
9605                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9606                                          inner_headers);
9607                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9608         } else {
9609                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9610                                          outer_headers);
9611                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9612         }
9613         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9614         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
9615         if (!icmp6_v)
9616                 return;
9617         if (!icmp6_m)
9618                 icmp6_m = &rte_flow_item_icmp6_mask;
9619         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
9620         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
9621                  icmp6_v->type & icmp6_m->type);
9622         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
9623         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
9624                  icmp6_v->code & icmp6_m->code);
9625 }
9626
9627 /**
9628  * Add ICMP item to matcher and to the value.
9629  *
9630  * @param[in, out] matcher
9631  *   Flow matcher.
9632  * @param[in, out] key
9633  *   Flow matcher value.
9634  * @param[in] item
9635  *   Flow pattern to translate.
9636  * @param[in] inner
9637  *   Item is inner pattern.
9638  */
9639 static void
9640 flow_dv_translate_item_icmp(void *matcher, void *key,
9641                             const struct rte_flow_item *item,
9642                             int inner)
9643 {
9644         const struct rte_flow_item_icmp *icmp_m = item->mask;
9645         const struct rte_flow_item_icmp *icmp_v = item->spec;
9646         uint32_t icmp_header_data_m = 0;
9647         uint32_t icmp_header_data_v = 0;
9648         void *headers_m;
9649         void *headers_v;
9650         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9651                                      misc_parameters_3);
9652         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9653         if (inner) {
9654                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9655                                          inner_headers);
9656                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9657         } else {
9658                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9659                                          outer_headers);
9660                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9661         }
9662         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9663         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
9664         if (!icmp_v)
9665                 return;
9666         if (!icmp_m)
9667                 icmp_m = &rte_flow_item_icmp_mask;
9668         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
9669                  icmp_m->hdr.icmp_type);
9670         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
9671                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
9672         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
9673                  icmp_m->hdr.icmp_code);
9674         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
9675                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
9676         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
9677         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
9678         if (icmp_header_data_m) {
9679                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
9680                 icmp_header_data_v |=
9681                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
9682                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
9683                          icmp_header_data_m);
9684                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
9685                          icmp_header_data_v & icmp_header_data_m);
9686         }
9687 }
9688
9689 /**
9690  * Add GTP item to matcher and to the value.
9691  *
9692  * @param[in, out] matcher
9693  *   Flow matcher.
9694  * @param[in, out] key
9695  *   Flow matcher value.
9696  * @param[in] item
9697  *   Flow pattern to translate.
9698  * @param[in] inner
9699  *   Item is inner pattern.
9700  */
9701 static void
9702 flow_dv_translate_item_gtp(void *matcher, void *key,
9703                            const struct rte_flow_item *item, int inner)
9704 {
9705         const struct rte_flow_item_gtp *gtp_m = item->mask;
9706         const struct rte_flow_item_gtp *gtp_v = item->spec;
9707         void *headers_m;
9708         void *headers_v;
9709         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9710                                      misc_parameters_3);
9711         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9712         uint16_t dport = RTE_GTPU_UDP_PORT;
9713
9714         if (inner) {
9715                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9716                                          inner_headers);
9717                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9718         } else {
9719                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9720                                          outer_headers);
9721                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9722         }
9723         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9724                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9725                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9726         }
9727         if (!gtp_v)
9728                 return;
9729         if (!gtp_m)
9730                 gtp_m = &rte_flow_item_gtp_mask;
9731         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
9732                  gtp_m->v_pt_rsv_flags);
9733         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
9734                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
9735         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
9736         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
9737                  gtp_v->msg_type & gtp_m->msg_type);
9738         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
9739                  rte_be_to_cpu_32(gtp_m->teid));
9740         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
9741                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
9742 }
9743
9744 /**
9745  * Add GTP PSC item to matcher.
9746  *
9747  * @param[in, out] matcher
9748  *   Flow matcher.
9749  * @param[in, out] key
9750  *   Flow matcher value.
9751  * @param[in] item
9752  *   Flow pattern to translate.
9753  */
9754 static int
9755 flow_dv_translate_item_gtp_psc(void *matcher, void *key,
9756                                const struct rte_flow_item *item)
9757 {
9758         const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask;
9759         const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec;
9760         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9761                         misc_parameters_3);
9762         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9763         union {
9764                 uint32_t w32;
9765                 struct {
9766                         uint16_t seq_num;
9767                         uint8_t npdu_num;
9768                         uint8_t next_ext_header_type;
9769                 };
9770         } dw_2;
9771         uint8_t gtp_flags;
9772
9773         /* Always set E-flag match on one, regardless of GTP item settings. */
9774         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags);
9775         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9776         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags);
9777         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags);
9778         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9779         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags);
9780         /*Set next extension header type. */
9781         dw_2.seq_num = 0;
9782         dw_2.npdu_num = 0;
9783         dw_2.next_ext_header_type = 0xff;
9784         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2,
9785                  rte_cpu_to_be_32(dw_2.w32));
9786         dw_2.seq_num = 0;
9787         dw_2.npdu_num = 0;
9788         dw_2.next_ext_header_type = 0x85;
9789         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2,
9790                  rte_cpu_to_be_32(dw_2.w32));
9791         if (gtp_psc_v) {
9792                 union {
9793                         uint32_t w32;
9794                         struct {
9795                                 uint8_t len;
9796                                 uint8_t type_flags;
9797                                 uint8_t qfi;
9798                                 uint8_t reserved;
9799                         };
9800                 } dw_0;
9801
9802                 /*Set extension header PDU type and Qos. */
9803                 if (!gtp_psc_m)
9804                         gtp_psc_m = &rte_flow_item_gtp_psc_mask;
9805                 dw_0.w32 = 0;
9806                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->pdu_type);
9807                 dw_0.qfi = gtp_psc_m->qfi;
9808                 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0,
9809                          rte_cpu_to_be_32(dw_0.w32));
9810                 dw_0.w32 = 0;
9811                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->pdu_type &
9812                                                         gtp_psc_m->pdu_type);
9813                 dw_0.qfi = gtp_psc_v->qfi & gtp_psc_m->qfi;
9814                 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0,
9815                          rte_cpu_to_be_32(dw_0.w32));
9816         }
9817         return 0;
9818 }
9819
9820 /**
9821  * Add eCPRI item to matcher and to the value.
9822  *
9823  * @param[in] dev
9824  *   The devich to configure through.
9825  * @param[in, out] matcher
9826  *   Flow matcher.
9827  * @param[in, out] key
9828  *   Flow matcher value.
9829  * @param[in] item
9830  *   Flow pattern to translate.
9831  * @param[in] samples
9832  *   Sample IDs to be used in the matching.
9833  */
9834 static void
9835 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
9836                              void *key, const struct rte_flow_item *item)
9837 {
9838         struct mlx5_priv *priv = dev->data->dev_private;
9839         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
9840         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
9841         struct rte_ecpri_common_hdr common;
9842         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
9843                                      misc_parameters_4);
9844         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
9845         uint32_t *samples;
9846         void *dw_m;
9847         void *dw_v;
9848
9849         if (!ecpri_v)
9850                 return;
9851         if (!ecpri_m)
9852                 ecpri_m = &rte_flow_item_ecpri_mask;
9853         /*
9854          * Maximal four DW samples are supported in a single matching now.
9855          * Two are used now for a eCPRI matching:
9856          * 1. Type: one byte, mask should be 0x00ff0000 in network order
9857          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
9858          *    if any.
9859          */
9860         if (!ecpri_m->hdr.common.u32)
9861                 return;
9862         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
9863         /* Need to take the whole DW as the mask to fill the entry. */
9864         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9865                             prog_sample_field_value_0);
9866         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9867                             prog_sample_field_value_0);
9868         /* Already big endian (network order) in the header. */
9869         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
9870         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32;
9871         /* Sample#0, used for matching type, offset 0. */
9872         MLX5_SET(fte_match_set_misc4, misc4_m,
9873                  prog_sample_field_id_0, samples[0]);
9874         /* It makes no sense to set the sample ID in the mask field. */
9875         MLX5_SET(fte_match_set_misc4, misc4_v,
9876                  prog_sample_field_id_0, samples[0]);
9877         /*
9878          * Checking if message body part needs to be matched.
9879          * Some wildcard rules only matching type field should be supported.
9880          */
9881         if (ecpri_m->hdr.dummy[0]) {
9882                 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32);
9883                 switch (common.type) {
9884                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
9885                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
9886                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
9887                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9888                                             prog_sample_field_value_1);
9889                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9890                                             prog_sample_field_value_1);
9891                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
9892                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] &
9893                                             ecpri_m->hdr.dummy[0];
9894                         /* Sample#1, to match message body, offset 4. */
9895                         MLX5_SET(fte_match_set_misc4, misc4_m,
9896                                  prog_sample_field_id_1, samples[1]);
9897                         MLX5_SET(fte_match_set_misc4, misc4_v,
9898                                  prog_sample_field_id_1, samples[1]);
9899                         break;
9900                 default:
9901                         /* Others, do not match any sample ID. */
9902                         break;
9903                 }
9904         }
9905 }
9906
9907 /*
9908  * Add connection tracking status item to matcher
9909  *
9910  * @param[in] dev
9911  *   The devich to configure through.
9912  * @param[in, out] matcher
9913  *   Flow matcher.
9914  * @param[in, out] key
9915  *   Flow matcher value.
9916  * @param[in] item
9917  *   Flow pattern to translate.
9918  */
9919 static void
9920 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev,
9921                               void *matcher, void *key,
9922                               const struct rte_flow_item *item)
9923 {
9924         uint32_t reg_value = 0;
9925         int reg_id;
9926         /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */
9927         uint32_t reg_mask = 0;
9928         const struct rte_flow_item_conntrack *spec = item->spec;
9929         const struct rte_flow_item_conntrack *mask = item->mask;
9930         uint32_t flags;
9931         struct rte_flow_error error;
9932
9933         if (!mask)
9934                 mask = &rte_flow_item_conntrack_mask;
9935         if (!spec || !mask->flags)
9936                 return;
9937         flags = spec->flags & mask->flags;
9938         /* The conflict should be checked in the validation. */
9939         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID)
9940                 reg_value |= MLX5_CT_SYNDROME_VALID;
9941         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
9942                 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE;
9943         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID)
9944                 reg_value |= MLX5_CT_SYNDROME_INVALID;
9945         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)
9946                 reg_value |= MLX5_CT_SYNDROME_TRAP;
9947         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
9948                 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET;
9949         if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID |
9950                            RTE_FLOW_CONNTRACK_PKT_STATE_INVALID |
9951                            RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))
9952                 reg_mask |= 0xc0;
9953         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
9954                 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE;
9955         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
9956                 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET;
9957         /* The REG_C_x value could be saved during startup. */
9958         reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error);
9959         if (reg_id == REG_NON)
9960                 return;
9961         flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id,
9962                                reg_value, reg_mask);
9963 }
9964
9965 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
9966
9967 #define HEADER_IS_ZERO(match_criteria, headers)                              \
9968         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
9969                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
9970
9971 /**
9972  * Calculate flow matcher enable bitmap.
9973  *
9974  * @param match_criteria
9975  *   Pointer to flow matcher criteria.
9976  *
9977  * @return
9978  *   Bitmap of enabled fields.
9979  */
9980 static uint8_t
9981 flow_dv_matcher_enable(uint32_t *match_criteria)
9982 {
9983         uint8_t match_criteria_enable;
9984
9985         match_criteria_enable =
9986                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
9987                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
9988         match_criteria_enable |=
9989                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
9990                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
9991         match_criteria_enable |=
9992                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
9993                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
9994         match_criteria_enable |=
9995                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
9996                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9997         match_criteria_enable |=
9998                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
9999                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
10000         match_criteria_enable |=
10001                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
10002                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
10003         match_criteria_enable |=
10004                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_5)) <<
10005                 MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT;
10006         return match_criteria_enable;
10007 }
10008
10009 static void
10010 __flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria)
10011 {
10012         /*
10013          * Check flow matching criteria first, subtract misc5/4 length if flow
10014          * doesn't own misc5/4 parameters. In some old rdma-core releases,
10015          * misc5/4 are not supported, and matcher creation failure is expected
10016          * w/o subtration. If misc5 is provided, misc4 must be counted in since
10017          * misc5 is right after misc4.
10018          */
10019         if (!(match_criteria & (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT))) {
10020                 *size = MLX5_ST_SZ_BYTES(fte_match_param) -
10021                         MLX5_ST_SZ_BYTES(fte_match_set_misc5);
10022                 if (!(match_criteria & (1 <<
10023                         MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT))) {
10024                         *size -= MLX5_ST_SZ_BYTES(fte_match_set_misc4);
10025                 }
10026         }
10027 }
10028
10029 static struct mlx5_list_entry *
10030 flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
10031                          struct mlx5_list_entry *entry, void *cb_ctx)
10032 {
10033         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10034         struct mlx5_flow_dv_matcher *ref = ctx->data;
10035         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10036                                                             typeof(*tbl), tbl);
10037         struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
10038                                                             sizeof(*resource),
10039                                                             0, SOCKET_ID_ANY);
10040
10041         if (!resource) {
10042                 rte_flow_error_set(ctx->error, ENOMEM,
10043                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10044                                    "cannot create matcher");
10045                 return NULL;
10046         }
10047         memcpy(resource, entry, sizeof(*resource));
10048         resource->tbl = &tbl->tbl;
10049         return &resource->entry;
10050 }
10051
10052 static void
10053 flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
10054                              struct mlx5_list_entry *entry)
10055 {
10056         mlx5_free(entry);
10057 }
10058
10059 struct mlx5_list_entry *
10060 flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
10061 {
10062         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10063         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10064         struct rte_eth_dev *dev = ctx->dev;
10065         struct mlx5_flow_tbl_data_entry *tbl_data;
10066         struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
10067         struct rte_flow_error *error = ctx->error;
10068         union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
10069         struct mlx5_flow_tbl_resource *tbl;
10070         void *domain;
10071         uint32_t idx = 0;
10072         int ret;
10073
10074         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10075         if (!tbl_data) {
10076                 rte_flow_error_set(error, ENOMEM,
10077                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10078                                    NULL,
10079                                    "cannot allocate flow table data entry");
10080                 return NULL;
10081         }
10082         tbl_data->idx = idx;
10083         tbl_data->tunnel = tt_prm->tunnel;
10084         tbl_data->group_id = tt_prm->group_id;
10085         tbl_data->external = !!tt_prm->external;
10086         tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
10087         tbl_data->is_egress = !!key.is_egress;
10088         tbl_data->is_transfer = !!key.is_fdb;
10089         tbl_data->dummy = !!key.dummy;
10090         tbl_data->level = key.level;
10091         tbl_data->id = key.id;
10092         tbl = &tbl_data->tbl;
10093         if (key.dummy)
10094                 return &tbl_data->entry;
10095         if (key.is_fdb)
10096                 domain = sh->fdb_domain;
10097         else if (key.is_egress)
10098                 domain = sh->tx_domain;
10099         else
10100                 domain = sh->rx_domain;
10101         ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
10102         if (ret) {
10103                 rte_flow_error_set(error, ENOMEM,
10104                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10105                                    NULL, "cannot create flow table object");
10106                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10107                 return NULL;
10108         }
10109         if (key.level != 0) {
10110                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
10111                                         (tbl->obj, &tbl_data->jump.action);
10112                 if (ret) {
10113                         rte_flow_error_set(error, ENOMEM,
10114                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10115                                            NULL,
10116                                            "cannot create flow jump action");
10117                         mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10118                         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10119                         return NULL;
10120                 }
10121         }
10122         MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
10123               key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
10124               key.level, key.id);
10125         tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
10126                                               flow_dv_matcher_create_cb,
10127                                               flow_dv_matcher_match_cb,
10128                                               flow_dv_matcher_remove_cb,
10129                                               flow_dv_matcher_clone_cb,
10130                                               flow_dv_matcher_clone_free_cb);
10131         if (!tbl_data->matchers) {
10132                 rte_flow_error_set(error, ENOMEM,
10133                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10134                                    NULL,
10135                                    "cannot create tbl matcher list");
10136                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10137                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10138                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10139                 return NULL;
10140         }
10141         return &tbl_data->entry;
10142 }
10143
10144 int
10145 flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10146                      void *cb_ctx)
10147 {
10148         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10149         struct mlx5_flow_tbl_data_entry *tbl_data =
10150                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10151         union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
10152
10153         return tbl_data->level != key.level ||
10154                tbl_data->id != key.id ||
10155                tbl_data->dummy != key.dummy ||
10156                tbl_data->is_transfer != !!key.is_fdb ||
10157                tbl_data->is_egress != !!key.is_egress;
10158 }
10159
10160 struct mlx5_list_entry *
10161 flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10162                       void *cb_ctx)
10163 {
10164         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10165         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10166         struct mlx5_flow_tbl_data_entry *tbl_data;
10167         struct rte_flow_error *error = ctx->error;
10168         uint32_t idx = 0;
10169
10170         tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10171         if (!tbl_data) {
10172                 rte_flow_error_set(error, ENOMEM,
10173                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10174                                    NULL,
10175                                    "cannot allocate flow table data entry");
10176                 return NULL;
10177         }
10178         memcpy(tbl_data, oentry, sizeof(*tbl_data));
10179         tbl_data->idx = idx;
10180         return &tbl_data->entry;
10181 }
10182
10183 void
10184 flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10185 {
10186         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10187         struct mlx5_flow_tbl_data_entry *tbl_data =
10188                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10189
10190         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10191 }
10192
10193 /**
10194  * Get a flow table.
10195  *
10196  * @param[in, out] dev
10197  *   Pointer to rte_eth_dev structure.
10198  * @param[in] table_level
10199  *   Table level to use.
10200  * @param[in] egress
10201  *   Direction of the table.
10202  * @param[in] transfer
10203  *   E-Switch or NIC flow.
10204  * @param[in] dummy
10205  *   Dummy entry for dv API.
10206  * @param[in] table_id
10207  *   Table id to use.
10208  * @param[out] error
10209  *   pointer to error structure.
10210  *
10211  * @return
10212  *   Returns tables resource based on the index, NULL in case of failed.
10213  */
10214 struct mlx5_flow_tbl_resource *
10215 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
10216                          uint32_t table_level, uint8_t egress,
10217                          uint8_t transfer,
10218                          bool external,
10219                          const struct mlx5_flow_tunnel *tunnel,
10220                          uint32_t group_id, uint8_t dummy,
10221                          uint32_t table_id,
10222                          struct rte_flow_error *error)
10223 {
10224         struct mlx5_priv *priv = dev->data->dev_private;
10225         union mlx5_flow_tbl_key table_key = {
10226                 {
10227                         .level = table_level,
10228                         .id = table_id,
10229                         .reserved = 0,
10230                         .dummy = !!dummy,
10231                         .is_fdb = !!transfer,
10232                         .is_egress = !!egress,
10233                 }
10234         };
10235         struct mlx5_flow_tbl_tunnel_prm tt_prm = {
10236                 .tunnel = tunnel,
10237                 .group_id = group_id,
10238                 .external = external,
10239         };
10240         struct mlx5_flow_cb_ctx ctx = {
10241                 .dev = dev,
10242                 .error = error,
10243                 .data = &table_key.v64,
10244                 .data2 = &tt_prm,
10245         };
10246         struct mlx5_list_entry *entry;
10247         struct mlx5_flow_tbl_data_entry *tbl_data;
10248
10249         entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
10250         if (!entry) {
10251                 rte_flow_error_set(error, ENOMEM,
10252                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10253                                    "cannot get table");
10254                 return NULL;
10255         }
10256         DRV_LOG(DEBUG, "table_level %u table_id %u "
10257                 "tunnel %u group %u registered.",
10258                 table_level, table_id,
10259                 tunnel ? tunnel->tunnel_id : 0, group_id);
10260         tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10261         return &tbl_data->tbl;
10262 }
10263
10264 void
10265 flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10266 {
10267         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10268         struct mlx5_flow_tbl_data_entry *tbl_data =
10269                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10270
10271         MLX5_ASSERT(entry && sh);
10272         if (tbl_data->jump.action)
10273                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10274         if (tbl_data->tbl.obj)
10275                 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
10276         if (tbl_data->tunnel_offload && tbl_data->external) {
10277                 struct mlx5_list_entry *he;
10278                 struct mlx5_hlist *tunnel_grp_hash;
10279                 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10280                 union tunnel_tbl_key tunnel_key = {
10281                         .tunnel_id = tbl_data->tunnel ?
10282                                         tbl_data->tunnel->tunnel_id : 0,
10283                         .group = tbl_data->group_id
10284                 };
10285                 uint32_t table_level = tbl_data->level;
10286                 struct mlx5_flow_cb_ctx ctx = {
10287                         .data = (void *)&tunnel_key.val,
10288                 };
10289
10290                 tunnel_grp_hash = tbl_data->tunnel ?
10291                                         tbl_data->tunnel->groups :
10292                                         thub->groups;
10293                 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
10294                 if (he)
10295                         mlx5_hlist_unregister(tunnel_grp_hash, he);
10296                 DRV_LOG(DEBUG,
10297                         "table_level %u id %u tunnel %u group %u released.",
10298                         table_level,
10299                         tbl_data->id,
10300                         tbl_data->tunnel ?
10301                         tbl_data->tunnel->tunnel_id : 0,
10302                         tbl_data->group_id);
10303         }
10304         mlx5_list_destroy(tbl_data->matchers);
10305         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10306 }
10307
10308 /**
10309  * Release a flow table.
10310  *
10311  * @param[in] sh
10312  *   Pointer to device shared structure.
10313  * @param[in] tbl
10314  *   Table resource to be released.
10315  *
10316  * @return
10317  *   Returns 0 if table was released, else return 1;
10318  */
10319 static int
10320 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
10321                              struct mlx5_flow_tbl_resource *tbl)
10322 {
10323         struct mlx5_flow_tbl_data_entry *tbl_data =
10324                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10325
10326         if (!tbl)
10327                 return 0;
10328         return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
10329 }
10330
10331 int
10332 flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
10333                          struct mlx5_list_entry *entry, void *cb_ctx)
10334 {
10335         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10336         struct mlx5_flow_dv_matcher *ref = ctx->data;
10337         struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur),
10338                                                         entry);
10339
10340         return cur->crc != ref->crc ||
10341                cur->priority != ref->priority ||
10342                memcmp((const void *)cur->mask.buf,
10343                       (const void *)ref->mask.buf, ref->mask.size);
10344 }
10345
10346 struct mlx5_list_entry *
10347 flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
10348 {
10349         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10350         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10351         struct mlx5_flow_dv_matcher *ref = ctx->data;
10352         struct mlx5_flow_dv_matcher *resource;
10353         struct mlx5dv_flow_matcher_attr dv_attr = {
10354                 .type = IBV_FLOW_ATTR_NORMAL,
10355                 .match_mask = (void *)&ref->mask,
10356         };
10357         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10358                                                             typeof(*tbl), tbl);
10359         int ret;
10360
10361         resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
10362                                SOCKET_ID_ANY);
10363         if (!resource) {
10364                 rte_flow_error_set(ctx->error, ENOMEM,
10365                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10366                                    "cannot create matcher");
10367                 return NULL;
10368         }
10369         *resource = *ref;
10370         dv_attr.match_criteria_enable =
10371                 flow_dv_matcher_enable(resource->mask.buf);
10372         __flow_dv_adjust_buf_size(&ref->mask.size,
10373                                   dv_attr.match_criteria_enable);
10374         dv_attr.priority = ref->priority;
10375         if (tbl->is_egress)
10376                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
10377         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
10378                                                &resource->matcher_object);
10379         if (ret) {
10380                 mlx5_free(resource);
10381                 rte_flow_error_set(ctx->error, ENOMEM,
10382                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10383                                    "cannot create matcher");
10384                 return NULL;
10385         }
10386         return &resource->entry;
10387 }
10388
10389 /**
10390  * Register the flow matcher.
10391  *
10392  * @param[in, out] dev
10393  *   Pointer to rte_eth_dev structure.
10394  * @param[in, out] matcher
10395  *   Pointer to flow matcher.
10396  * @param[in, out] key
10397  *   Pointer to flow table key.
10398  * @parm[in, out] dev_flow
10399  *   Pointer to the dev_flow.
10400  * @param[out] error
10401  *   pointer to error structure.
10402  *
10403  * @return
10404  *   0 on success otherwise -errno and errno is set.
10405  */
10406 static int
10407 flow_dv_matcher_register(struct rte_eth_dev *dev,
10408                          struct mlx5_flow_dv_matcher *ref,
10409                          union mlx5_flow_tbl_key *key,
10410                          struct mlx5_flow *dev_flow,
10411                          const struct mlx5_flow_tunnel *tunnel,
10412                          uint32_t group_id,
10413                          struct rte_flow_error *error)
10414 {
10415         struct mlx5_list_entry *entry;
10416         struct mlx5_flow_dv_matcher *resource;
10417         struct mlx5_flow_tbl_resource *tbl;
10418         struct mlx5_flow_tbl_data_entry *tbl_data;
10419         struct mlx5_flow_cb_ctx ctx = {
10420                 .error = error,
10421                 .data = ref,
10422         };
10423         /**
10424          * tunnel offload API requires this registration for cases when
10425          * tunnel match rule was inserted before tunnel set rule.
10426          */
10427         tbl = flow_dv_tbl_resource_get(dev, key->level,
10428                                        key->is_egress, key->is_fdb,
10429                                        dev_flow->external, tunnel,
10430                                        group_id, 0, key->id, error);
10431         if (!tbl)
10432                 return -rte_errno;      /* No need to refill the error info */
10433         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10434         ref->tbl = tbl;
10435         entry = mlx5_list_register(tbl_data->matchers, &ctx);
10436         if (!entry) {
10437                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
10438                 return rte_flow_error_set(error, ENOMEM,
10439                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10440                                           "cannot allocate ref memory");
10441         }
10442         resource = container_of(entry, typeof(*resource), entry);
10443         dev_flow->handle->dvh.matcher = resource;
10444         return 0;
10445 }
10446
10447 struct mlx5_list_entry *
10448 flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
10449 {
10450         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10451         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10452         struct mlx5_flow_dv_tag_resource *entry;
10453         uint32_t idx = 0;
10454         int ret;
10455
10456         entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10457         if (!entry) {
10458                 rte_flow_error_set(ctx->error, ENOMEM,
10459                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10460                                    "cannot allocate resource memory");
10461                 return NULL;
10462         }
10463         entry->idx = idx;
10464         entry->tag_id = *(uint32_t *)(ctx->data);
10465         ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
10466                                                   &entry->action);
10467         if (ret) {
10468                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
10469                 rte_flow_error_set(ctx->error, ENOMEM,
10470                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10471                                    NULL, "cannot create action");
10472                 return NULL;
10473         }
10474         return &entry->entry;
10475 }
10476
10477 int
10478 flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10479                      void *cb_ctx)
10480 {
10481         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10482         struct mlx5_flow_dv_tag_resource *tag =
10483                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10484
10485         return *(uint32_t *)(ctx->data) != tag->tag_id;
10486 }
10487
10488 struct mlx5_list_entry *
10489 flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10490                      void *cb_ctx)
10491 {
10492         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10493         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10494         struct mlx5_flow_dv_tag_resource *entry;
10495         uint32_t idx = 0;
10496
10497         entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10498         if (!entry) {
10499                 rte_flow_error_set(ctx->error, ENOMEM,
10500                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10501                                    "cannot allocate tag resource memory");
10502                 return NULL;
10503         }
10504         memcpy(entry, oentry, sizeof(*entry));
10505         entry->idx = idx;
10506         return &entry->entry;
10507 }
10508
10509 void
10510 flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10511 {
10512         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10513         struct mlx5_flow_dv_tag_resource *tag =
10514                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10515
10516         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10517 }
10518
10519 /**
10520  * Find existing tag resource or create and register a new one.
10521  *
10522  * @param dev[in, out]
10523  *   Pointer to rte_eth_dev structure.
10524  * @param[in, out] tag_be24
10525  *   Tag value in big endian then R-shift 8.
10526  * @parm[in, out] dev_flow
10527  *   Pointer to the dev_flow.
10528  * @param[out] error
10529  *   pointer to error structure.
10530  *
10531  * @return
10532  *   0 on success otherwise -errno and errno is set.
10533  */
10534 static int
10535 flow_dv_tag_resource_register
10536                         (struct rte_eth_dev *dev,
10537                          uint32_t tag_be24,
10538                          struct mlx5_flow *dev_flow,
10539                          struct rte_flow_error *error)
10540 {
10541         struct mlx5_priv *priv = dev->data->dev_private;
10542         struct mlx5_flow_dv_tag_resource *resource;
10543         struct mlx5_list_entry *entry;
10544         struct mlx5_flow_cb_ctx ctx = {
10545                                         .error = error,
10546                                         .data = &tag_be24,
10547                                         };
10548
10549         entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
10550         if (entry) {
10551                 resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
10552                                         entry);
10553                 dev_flow->handle->dvh.rix_tag = resource->idx;
10554                 dev_flow->dv.tag_resource = resource;
10555                 return 0;
10556         }
10557         return -rte_errno;
10558 }
10559
10560 void
10561 flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10562 {
10563         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10564         struct mlx5_flow_dv_tag_resource *tag =
10565                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10566
10567         MLX5_ASSERT(tag && sh && tag->action);
10568         claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
10569         DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag);
10570         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10571 }
10572
10573 /**
10574  * Release the tag.
10575  *
10576  * @param dev
10577  *   Pointer to Ethernet device.
10578  * @param tag_idx
10579  *   Tag index.
10580  *
10581  * @return
10582  *   1 while a reference on it exists, 0 when freed.
10583  */
10584 static int
10585 flow_dv_tag_release(struct rte_eth_dev *dev,
10586                     uint32_t tag_idx)
10587 {
10588         struct mlx5_priv *priv = dev->data->dev_private;
10589         struct mlx5_flow_dv_tag_resource *tag;
10590
10591         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
10592         if (!tag)
10593                 return 0;
10594         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
10595                 dev->data->port_id, (void *)tag, tag->entry.ref_cnt);
10596         return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry);
10597 }
10598
10599 /**
10600  * Translate port ID action to vport.
10601  *
10602  * @param[in] dev
10603  *   Pointer to rte_eth_dev structure.
10604  * @param[in] action
10605  *   Pointer to the port ID action.
10606  * @param[out] dst_port_id
10607  *   The target port ID.
10608  * @param[out] error
10609  *   Pointer to the error structure.
10610  *
10611  * @return
10612  *   0 on success, a negative errno value otherwise and rte_errno is set.
10613  */
10614 static int
10615 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
10616                                  const struct rte_flow_action *action,
10617                                  uint32_t *dst_port_id,
10618                                  struct rte_flow_error *error)
10619 {
10620         uint32_t port;
10621         struct mlx5_priv *priv;
10622         const struct rte_flow_action_port_id *conf =
10623                         (const struct rte_flow_action_port_id *)action->conf;
10624
10625         port = conf->original ? dev->data->port_id : conf->id;
10626         priv = mlx5_port_to_eswitch_info(port, false);
10627         if (!priv)
10628                 return rte_flow_error_set(error, -rte_errno,
10629                                           RTE_FLOW_ERROR_TYPE_ACTION,
10630                                           NULL,
10631                                           "No eswitch info was found for port");
10632 #ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT
10633         /*
10634          * This parameter is transferred to
10635          * mlx5dv_dr_action_create_dest_ib_port().
10636          */
10637         *dst_port_id = priv->dev_port;
10638 #else
10639         /*
10640          * Legacy mode, no LAG configurations is supported.
10641          * This parameter is transferred to
10642          * mlx5dv_dr_action_create_dest_vport().
10643          */
10644         *dst_port_id = priv->vport_id;
10645 #endif
10646         return 0;
10647 }
10648
10649 /**
10650  * Create a counter with aging configuration.
10651  *
10652  * @param[in] dev
10653  *   Pointer to rte_eth_dev structure.
10654  * @param[in] dev_flow
10655  *   Pointer to the mlx5_flow.
10656  * @param[out] count
10657  *   Pointer to the counter action configuration.
10658  * @param[in] age
10659  *   Pointer to the aging action configuration.
10660  *
10661  * @return
10662  *   Index to flow counter on success, 0 otherwise.
10663  */
10664 static uint32_t
10665 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
10666                                 struct mlx5_flow *dev_flow,
10667                                 const struct rte_flow_action_count *count,
10668                                 const struct rte_flow_action_age *age)
10669 {
10670         uint32_t counter;
10671         struct mlx5_age_param *age_param;
10672
10673         if (count && count->shared)
10674                 counter = flow_dv_counter_get_shared(dev, count->id);
10675         else
10676                 counter = flow_dv_counter_alloc(dev, !!age);
10677         if (!counter || age == NULL)
10678                 return counter;
10679         age_param = flow_dv_counter_idx_get_age(dev, counter);
10680         age_param->context = age->context ? age->context :
10681                 (void *)(uintptr_t)(dev_flow->flow_idx);
10682         age_param->timeout = age->timeout;
10683         age_param->port_id = dev->data->port_id;
10684         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
10685         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
10686         return counter;
10687 }
10688
10689 /**
10690  * Add Tx queue matcher
10691  *
10692  * @param[in] dev
10693  *   Pointer to the dev struct.
10694  * @param[in, out] matcher
10695  *   Flow matcher.
10696  * @param[in, out] key
10697  *   Flow matcher value.
10698  * @param[in] item
10699  *   Flow pattern to translate.
10700  * @param[in] inner
10701  *   Item is inner pattern.
10702  */
10703 static void
10704 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
10705                                 void *matcher, void *key,
10706                                 const struct rte_flow_item *item)
10707 {
10708         const struct mlx5_rte_flow_item_tx_queue *queue_m;
10709         const struct mlx5_rte_flow_item_tx_queue *queue_v;
10710         void *misc_m =
10711                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
10712         void *misc_v =
10713                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
10714         struct mlx5_txq_ctrl *txq;
10715         uint32_t queue;
10716
10717
10718         queue_m = (const void *)item->mask;
10719         if (!queue_m)
10720                 return;
10721         queue_v = (const void *)item->spec;
10722         if (!queue_v)
10723                 return;
10724         txq = mlx5_txq_get(dev, queue_v->queue);
10725         if (!txq)
10726                 return;
10727         queue = txq->obj->sq->id;
10728         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
10729         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
10730                  queue & queue_m->queue);
10731         mlx5_txq_release(dev, queue_v->queue);
10732 }
10733
10734 /**
10735  * Set the hash fields according to the @p flow information.
10736  *
10737  * @param[in] dev_flow
10738  *   Pointer to the mlx5_flow.
10739  * @param[in] rss_desc
10740  *   Pointer to the mlx5_flow_rss_desc.
10741  */
10742 static void
10743 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
10744                        struct mlx5_flow_rss_desc *rss_desc)
10745 {
10746         uint64_t items = dev_flow->handle->layers;
10747         int rss_inner = 0;
10748         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
10749
10750         dev_flow->hash_fields = 0;
10751 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
10752         if (rss_desc->level >= 2) {
10753                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
10754                 rss_inner = 1;
10755         }
10756 #endif
10757         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
10758             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
10759                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
10760                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10761                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
10762                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10763                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
10764                         else
10765                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
10766                 }
10767         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
10768                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
10769                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
10770                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10771                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
10772                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10773                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
10774                         else
10775                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
10776                 }
10777         }
10778         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
10779             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
10780                 if (rss_types & ETH_RSS_UDP) {
10781                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10782                                 dev_flow->hash_fields |=
10783                                                 IBV_RX_HASH_SRC_PORT_UDP;
10784                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10785                                 dev_flow->hash_fields |=
10786                                                 IBV_RX_HASH_DST_PORT_UDP;
10787                         else
10788                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
10789                 }
10790         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
10791                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
10792                 if (rss_types & ETH_RSS_TCP) {
10793                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10794                                 dev_flow->hash_fields |=
10795                                                 IBV_RX_HASH_SRC_PORT_TCP;
10796                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10797                                 dev_flow->hash_fields |=
10798                                                 IBV_RX_HASH_DST_PORT_TCP;
10799                         else
10800                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
10801                 }
10802         }
10803 }
10804
10805 /**
10806  * Prepare an Rx Hash queue.
10807  *
10808  * @param dev
10809  *   Pointer to Ethernet device.
10810  * @param[in] dev_flow
10811  *   Pointer to the mlx5_flow.
10812  * @param[in] rss_desc
10813  *   Pointer to the mlx5_flow_rss_desc.
10814  * @param[out] hrxq_idx
10815  *   Hash Rx queue index.
10816  *
10817  * @return
10818  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
10819  */
10820 static struct mlx5_hrxq *
10821 flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
10822                      struct mlx5_flow *dev_flow,
10823                      struct mlx5_flow_rss_desc *rss_desc,
10824                      uint32_t *hrxq_idx)
10825 {
10826         struct mlx5_priv *priv = dev->data->dev_private;
10827         struct mlx5_flow_handle *dh = dev_flow->handle;
10828         struct mlx5_hrxq *hrxq;
10829
10830         MLX5_ASSERT(rss_desc->queue_num);
10831         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
10832         rss_desc->hash_fields = dev_flow->hash_fields;
10833         rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL);
10834         rss_desc->shared_rss = 0;
10835         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
10836         if (!*hrxq_idx)
10837                 return NULL;
10838         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10839                               *hrxq_idx);
10840         return hrxq;
10841 }
10842
10843 /**
10844  * Release sample sub action resource.
10845  *
10846  * @param[in, out] dev
10847  *   Pointer to rte_eth_dev structure.
10848  * @param[in] act_res
10849  *   Pointer to sample sub action resource.
10850  */
10851 static void
10852 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
10853                                    struct mlx5_flow_sub_actions_idx *act_res)
10854 {
10855         if (act_res->rix_hrxq) {
10856                 mlx5_hrxq_release(dev, act_res->rix_hrxq);
10857                 act_res->rix_hrxq = 0;
10858         }
10859         if (act_res->rix_encap_decap) {
10860                 flow_dv_encap_decap_resource_release(dev,
10861                                                      act_res->rix_encap_decap);
10862                 act_res->rix_encap_decap = 0;
10863         }
10864         if (act_res->rix_port_id_action) {
10865                 flow_dv_port_id_action_resource_release(dev,
10866                                                 act_res->rix_port_id_action);
10867                 act_res->rix_port_id_action = 0;
10868         }
10869         if (act_res->rix_tag) {
10870                 flow_dv_tag_release(dev, act_res->rix_tag);
10871                 act_res->rix_tag = 0;
10872         }
10873         if (act_res->rix_jump) {
10874                 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump);
10875                 act_res->rix_jump = 0;
10876         }
10877 }
10878
10879 int
10880 flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
10881                         struct mlx5_list_entry *entry, void *cb_ctx)
10882 {
10883         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10884         struct rte_eth_dev *dev = ctx->dev;
10885         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
10886         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
10887                                                               typeof(*resource),
10888                                                               entry);
10889
10890         if (ctx_resource->ratio == resource->ratio &&
10891             ctx_resource->ft_type == resource->ft_type &&
10892             ctx_resource->ft_id == resource->ft_id &&
10893             ctx_resource->set_action == resource->set_action &&
10894             !memcmp((void *)&ctx_resource->sample_act,
10895                     (void *)&resource->sample_act,
10896                     sizeof(struct mlx5_flow_sub_actions_list))) {
10897                 /*
10898                  * Existing sample action should release the prepared
10899                  * sub-actions reference counter.
10900                  */
10901                 flow_dv_sample_sub_actions_release(dev,
10902                                                    &ctx_resource->sample_idx);
10903                 return 0;
10904         }
10905         return 1;
10906 }
10907
10908 struct mlx5_list_entry *
10909 flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
10910 {
10911         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10912         struct rte_eth_dev *dev = ctx->dev;
10913         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
10914         void **sample_dv_actions = ctx_resource->sub_actions;
10915         struct mlx5_flow_dv_sample_resource *resource;
10916         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
10917         struct mlx5_priv *priv = dev->data->dev_private;
10918         struct mlx5_dev_ctx_shared *sh = priv->sh;
10919         struct mlx5_flow_tbl_resource *tbl;
10920         uint32_t idx = 0;
10921         const uint32_t next_ft_step = 1;
10922         uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
10923         uint8_t is_egress = 0;
10924         uint8_t is_transfer = 0;
10925         struct rte_flow_error *error = ctx->error;
10926
10927         /* Register new sample resource. */
10928         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
10929         if (!resource) {
10930                 rte_flow_error_set(error, ENOMEM,
10931                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10932                                           NULL,
10933                                           "cannot allocate resource memory");
10934                 return NULL;
10935         }
10936         *resource = *ctx_resource;
10937         /* Create normal path table level */
10938         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
10939                 is_transfer = 1;
10940         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
10941                 is_egress = 1;
10942         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
10943                                         is_egress, is_transfer,
10944                                         true, NULL, 0, 0, 0, error);
10945         if (!tbl) {
10946                 rte_flow_error_set(error, ENOMEM,
10947                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10948                                           NULL,
10949                                           "fail to create normal path table "
10950                                           "for sample");
10951                 goto error;
10952         }
10953         resource->normal_path_tbl = tbl;
10954         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10955                 if (!sh->default_miss_action) {
10956                         rte_flow_error_set(error, ENOMEM,
10957                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10958                                                 NULL,
10959                                                 "default miss action was not "
10960                                                 "created");
10961                         goto error;
10962                 }
10963                 sample_dv_actions[ctx_resource->sample_act.actions_num++] =
10964                                                 sh->default_miss_action;
10965         }
10966         /* Create a DR sample action */
10967         sampler_attr.sample_ratio = resource->ratio;
10968         sampler_attr.default_next_table = tbl->obj;
10969         sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
10970         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
10971                                                         &sample_dv_actions[0];
10972         sampler_attr.action = resource->set_action;
10973         if (mlx5_os_flow_dr_create_flow_action_sampler
10974                         (&sampler_attr, &resource->verbs_action)) {
10975                 rte_flow_error_set(error, ENOMEM,
10976                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10977                                         NULL, "cannot create sample action");
10978                 goto error;
10979         }
10980         resource->idx = idx;
10981         resource->dev = dev;
10982         return &resource->entry;
10983 error:
10984         if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
10985                 flow_dv_sample_sub_actions_release(dev,
10986                                                    &resource->sample_idx);
10987         if (resource->normal_path_tbl)
10988                 flow_dv_tbl_resource_release(MLX5_SH(dev),
10989                                 resource->normal_path_tbl);
10990         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10991         return NULL;
10992
10993 }
10994
10995 struct mlx5_list_entry *
10996 flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
10997                          struct mlx5_list_entry *entry __rte_unused,
10998                          void *cb_ctx)
10999 {
11000         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11001         struct rte_eth_dev *dev = ctx->dev;
11002         struct mlx5_flow_dv_sample_resource *resource;
11003         struct mlx5_priv *priv = dev->data->dev_private;
11004         struct mlx5_dev_ctx_shared *sh = priv->sh;
11005         uint32_t idx = 0;
11006
11007         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11008         if (!resource) {
11009                 rte_flow_error_set(ctx->error, ENOMEM,
11010                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11011                                           NULL,
11012                                           "cannot allocate resource memory");
11013                 return NULL;
11014         }
11015         memcpy(resource, entry, sizeof(*resource));
11016         resource->idx = idx;
11017         resource->dev = dev;
11018         return &resource->entry;
11019 }
11020
11021 void
11022 flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
11023                              struct mlx5_list_entry *entry)
11024 {
11025         struct mlx5_flow_dv_sample_resource *resource =
11026                                   container_of(entry, typeof(*resource), entry);
11027         struct rte_eth_dev *dev = resource->dev;
11028         struct mlx5_priv *priv = dev->data->dev_private;
11029
11030         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
11031 }
11032
11033 /**
11034  * Find existing sample resource or create and register a new one.
11035  *
11036  * @param[in, out] dev
11037  *   Pointer to rte_eth_dev structure.
11038  * @param[in] ref
11039  *   Pointer to sample resource reference.
11040  * @parm[in, out] dev_flow
11041  *   Pointer to the dev_flow.
11042  * @param[out] error
11043  *   pointer to error structure.
11044  *
11045  * @return
11046  *   0 on success otherwise -errno and errno is set.
11047  */
11048 static int
11049 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
11050                          struct mlx5_flow_dv_sample_resource *ref,
11051                          struct mlx5_flow *dev_flow,
11052                          struct rte_flow_error *error)
11053 {
11054         struct mlx5_flow_dv_sample_resource *resource;
11055         struct mlx5_list_entry *entry;
11056         struct mlx5_priv *priv = dev->data->dev_private;
11057         struct mlx5_flow_cb_ctx ctx = {
11058                 .dev = dev,
11059                 .error = error,
11060                 .data = ref,
11061         };
11062
11063         entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
11064         if (!entry)
11065                 return -rte_errno;
11066         resource = container_of(entry, typeof(*resource), entry);
11067         dev_flow->handle->dvh.rix_sample = resource->idx;
11068         dev_flow->dv.sample_res = resource;
11069         return 0;
11070 }
11071
11072 int
11073 flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
11074                             struct mlx5_list_entry *entry, void *cb_ctx)
11075 {
11076         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11077         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11078         struct rte_eth_dev *dev = ctx->dev;
11079         struct mlx5_flow_dv_dest_array_resource *resource =
11080                                   container_of(entry, typeof(*resource), entry);
11081         uint32_t idx = 0;
11082
11083         if (ctx_resource->num_of_dest == resource->num_of_dest &&
11084             ctx_resource->ft_type == resource->ft_type &&
11085             !memcmp((void *)resource->sample_act,
11086                     (void *)ctx_resource->sample_act,
11087                    (ctx_resource->num_of_dest *
11088                    sizeof(struct mlx5_flow_sub_actions_list)))) {
11089                 /*
11090                  * Existing sample action should release the prepared
11091                  * sub-actions reference counter.
11092                  */
11093                 for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11094                         flow_dv_sample_sub_actions_release(dev,
11095                                         &ctx_resource->sample_idx[idx]);
11096                 return 0;
11097         }
11098         return 1;
11099 }
11100
11101 struct mlx5_list_entry *
11102 flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11103 {
11104         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11105         struct rte_eth_dev *dev = ctx->dev;
11106         struct mlx5_flow_dv_dest_array_resource *resource;
11107         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11108         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
11109         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
11110         struct mlx5_priv *priv = dev->data->dev_private;
11111         struct mlx5_dev_ctx_shared *sh = priv->sh;
11112         struct mlx5_flow_sub_actions_list *sample_act;
11113         struct mlx5dv_dr_domain *domain;
11114         uint32_t idx = 0, res_idx = 0;
11115         struct rte_flow_error *error = ctx->error;
11116         uint64_t action_flags;
11117         int ret;
11118
11119         /* Register new destination array resource. */
11120         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11121                                             &res_idx);
11122         if (!resource) {
11123                 rte_flow_error_set(error, ENOMEM,
11124                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11125                                           NULL,
11126                                           "cannot allocate resource memory");
11127                 return NULL;
11128         }
11129         *resource = *ctx_resource;
11130         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11131                 domain = sh->fdb_domain;
11132         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
11133                 domain = sh->rx_domain;
11134         else
11135                 domain = sh->tx_domain;
11136         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11137                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
11138                                  mlx5_malloc(MLX5_MEM_ZERO,
11139                                  sizeof(struct mlx5dv_dr_action_dest_attr),
11140                                  0, SOCKET_ID_ANY);
11141                 if (!dest_attr[idx]) {
11142                         rte_flow_error_set(error, ENOMEM,
11143                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11144                                            NULL,
11145                                            "cannot allocate resource memory");
11146                         goto error;
11147                 }
11148                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
11149                 sample_act = &ctx_resource->sample_act[idx];
11150                 action_flags = sample_act->action_flags;
11151                 switch (action_flags) {
11152                 case MLX5_FLOW_ACTION_QUEUE:
11153                         dest_attr[idx]->dest = sample_act->dr_queue_action;
11154                         break;
11155                 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP):
11156                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
11157                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
11158                         dest_attr[idx]->dest_reformat->reformat =
11159                                         sample_act->dr_encap_action;
11160                         dest_attr[idx]->dest_reformat->dest =
11161                                         sample_act->dr_port_id_action;
11162                         break;
11163                 case MLX5_FLOW_ACTION_PORT_ID:
11164                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
11165                         break;
11166                 case MLX5_FLOW_ACTION_JUMP:
11167                         dest_attr[idx]->dest = sample_act->dr_jump_action;
11168                         break;
11169                 default:
11170                         rte_flow_error_set(error, EINVAL,
11171                                            RTE_FLOW_ERROR_TYPE_ACTION,
11172                                            NULL,
11173                                            "unsupported actions type");
11174                         goto error;
11175                 }
11176         }
11177         /* create a dest array actioin */
11178         ret = mlx5_os_flow_dr_create_flow_action_dest_array
11179                                                 (domain,
11180                                                  resource->num_of_dest,
11181                                                  dest_attr,
11182                                                  &resource->action);
11183         if (ret) {
11184                 rte_flow_error_set(error, ENOMEM,
11185                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11186                                    NULL,
11187                                    "cannot create destination array action");
11188                 goto error;
11189         }
11190         resource->idx = res_idx;
11191         resource->dev = dev;
11192         for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11193                 mlx5_free(dest_attr[idx]);
11194         return &resource->entry;
11195 error:
11196         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11197                 flow_dv_sample_sub_actions_release(dev,
11198                                                    &resource->sample_idx[idx]);
11199                 if (dest_attr[idx])
11200                         mlx5_free(dest_attr[idx]);
11201         }
11202         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
11203         return NULL;
11204 }
11205
11206 struct mlx5_list_entry *
11207 flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
11208                             struct mlx5_list_entry *entry __rte_unused,
11209                             void *cb_ctx)
11210 {
11211         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11212         struct rte_eth_dev *dev = ctx->dev;
11213         struct mlx5_flow_dv_dest_array_resource *resource;
11214         struct mlx5_priv *priv = dev->data->dev_private;
11215         struct mlx5_dev_ctx_shared *sh = priv->sh;
11216         uint32_t res_idx = 0;
11217         struct rte_flow_error *error = ctx->error;
11218
11219         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11220                                       &res_idx);
11221         if (!resource) {
11222                 rte_flow_error_set(error, ENOMEM,
11223                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11224                                           NULL,
11225                                           "cannot allocate dest-array memory");
11226                 return NULL;
11227         }
11228         memcpy(resource, entry, sizeof(*resource));
11229         resource->idx = res_idx;
11230         resource->dev = dev;
11231         return &resource->entry;
11232 }
11233
11234 void
11235 flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
11236                                  struct mlx5_list_entry *entry)
11237 {
11238         struct mlx5_flow_dv_dest_array_resource *resource =
11239                         container_of(entry, typeof(*resource), entry);
11240         struct rte_eth_dev *dev = resource->dev;
11241         struct mlx5_priv *priv = dev->data->dev_private;
11242
11243         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
11244 }
11245
11246 /**
11247  * Find existing destination array resource or create and register a new one.
11248  *
11249  * @param[in, out] dev
11250  *   Pointer to rte_eth_dev structure.
11251  * @param[in] ref
11252  *   Pointer to destination array resource reference.
11253  * @parm[in, out] dev_flow
11254  *   Pointer to the dev_flow.
11255  * @param[out] error
11256  *   pointer to error structure.
11257  *
11258  * @return
11259  *   0 on success otherwise -errno and errno is set.
11260  */
11261 static int
11262 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
11263                          struct mlx5_flow_dv_dest_array_resource *ref,
11264                          struct mlx5_flow *dev_flow,
11265                          struct rte_flow_error *error)
11266 {
11267         struct mlx5_flow_dv_dest_array_resource *resource;
11268         struct mlx5_priv *priv = dev->data->dev_private;
11269         struct mlx5_list_entry *entry;
11270         struct mlx5_flow_cb_ctx ctx = {
11271                 .dev = dev,
11272                 .error = error,
11273                 .data = ref,
11274         };
11275
11276         entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
11277         if (!entry)
11278                 return -rte_errno;
11279         resource = container_of(entry, typeof(*resource), entry);
11280         dev_flow->handle->dvh.rix_dest_array = resource->idx;
11281         dev_flow->dv.dest_array_res = resource;
11282         return 0;
11283 }
11284
11285 /**
11286  * Convert Sample action to DV specification.
11287  *
11288  * @param[in] dev
11289  *   Pointer to rte_eth_dev structure.
11290  * @param[in] action
11291  *   Pointer to sample action structure.
11292  * @param[in, out] dev_flow
11293  *   Pointer to the mlx5_flow.
11294  * @param[in] attr
11295  *   Pointer to the flow attributes.
11296  * @param[in, out] num_of_dest
11297  *   Pointer to the num of destination.
11298  * @param[in, out] sample_actions
11299  *   Pointer to sample actions list.
11300  * @param[in, out] res
11301  *   Pointer to sample resource.
11302  * @param[out] error
11303  *   Pointer to the error structure.
11304  *
11305  * @return
11306  *   0 on success, a negative errno value otherwise and rte_errno is set.
11307  */
11308 static int
11309 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
11310                                 const struct rte_flow_action_sample *action,
11311                                 struct mlx5_flow *dev_flow,
11312                                 const struct rte_flow_attr *attr,
11313                                 uint32_t *num_of_dest,
11314                                 void **sample_actions,
11315                                 struct mlx5_flow_dv_sample_resource *res,
11316                                 struct rte_flow_error *error)
11317 {
11318         struct mlx5_priv *priv = dev->data->dev_private;
11319         const struct rte_flow_action *sub_actions;
11320         struct mlx5_flow_sub_actions_list *sample_act;
11321         struct mlx5_flow_sub_actions_idx *sample_idx;
11322         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11323         struct rte_flow *flow = dev_flow->flow;
11324         struct mlx5_flow_rss_desc *rss_desc;
11325         uint64_t action_flags = 0;
11326
11327         MLX5_ASSERT(wks);
11328         rss_desc = &wks->rss_desc;
11329         sample_act = &res->sample_act;
11330         sample_idx = &res->sample_idx;
11331         res->ratio = action->ratio;
11332         sub_actions = action->actions;
11333         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
11334                 int type = sub_actions->type;
11335                 uint32_t pre_rix = 0;
11336                 void *pre_r;
11337                 switch (type) {
11338                 case RTE_FLOW_ACTION_TYPE_QUEUE:
11339                 {
11340                         const struct rte_flow_action_queue *queue;
11341                         struct mlx5_hrxq *hrxq;
11342                         uint32_t hrxq_idx;
11343
11344                         queue = sub_actions->conf;
11345                         rss_desc->queue_num = 1;
11346                         rss_desc->queue[0] = queue->index;
11347                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11348                                                     rss_desc, &hrxq_idx);
11349                         if (!hrxq)
11350                                 return rte_flow_error_set
11351                                         (error, rte_errno,
11352                                          RTE_FLOW_ERROR_TYPE_ACTION,
11353                                          NULL,
11354                                          "cannot create fate queue");
11355                         sample_act->dr_queue_action = hrxq->action;
11356                         sample_idx->rix_hrxq = hrxq_idx;
11357                         sample_actions[sample_act->actions_num++] =
11358                                                 hrxq->action;
11359                         (*num_of_dest)++;
11360                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
11361                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11362                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11363                         dev_flow->handle->fate_action =
11364                                         MLX5_FLOW_FATE_QUEUE;
11365                         break;
11366                 }
11367                 case RTE_FLOW_ACTION_TYPE_RSS:
11368                 {
11369                         struct mlx5_hrxq *hrxq;
11370                         uint32_t hrxq_idx;
11371                         const struct rte_flow_action_rss *rss;
11372                         const uint8_t *rss_key;
11373
11374                         rss = sub_actions->conf;
11375                         memcpy(rss_desc->queue, rss->queue,
11376                                rss->queue_num * sizeof(uint16_t));
11377                         rss_desc->queue_num = rss->queue_num;
11378                         /* NULL RSS key indicates default RSS key. */
11379                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11380                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11381                         /*
11382                          * rss->level and rss.types should be set in advance
11383                          * when expanding items for RSS.
11384                          */
11385                         flow_dv_hashfields_set(dev_flow, rss_desc);
11386                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11387                                                     rss_desc, &hrxq_idx);
11388                         if (!hrxq)
11389                                 return rte_flow_error_set
11390                                         (error, rte_errno,
11391                                          RTE_FLOW_ERROR_TYPE_ACTION,
11392                                          NULL,
11393                                          "cannot create fate queue");
11394                         sample_act->dr_queue_action = hrxq->action;
11395                         sample_idx->rix_hrxq = hrxq_idx;
11396                         sample_actions[sample_act->actions_num++] =
11397                                                 hrxq->action;
11398                         (*num_of_dest)++;
11399                         action_flags |= MLX5_FLOW_ACTION_RSS;
11400                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11401                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11402                         dev_flow->handle->fate_action =
11403                                         MLX5_FLOW_FATE_QUEUE;
11404                         break;
11405                 }
11406                 case RTE_FLOW_ACTION_TYPE_MARK:
11407                 {
11408                         uint32_t tag_be = mlx5_flow_mark_set
11409                                 (((const struct rte_flow_action_mark *)
11410                                 (sub_actions->conf))->id);
11411
11412                         dev_flow->handle->mark = 1;
11413                         pre_rix = dev_flow->handle->dvh.rix_tag;
11414                         /* Save the mark resource before sample */
11415                         pre_r = dev_flow->dv.tag_resource;
11416                         if (flow_dv_tag_resource_register(dev, tag_be,
11417                                                   dev_flow, error))
11418                                 return -rte_errno;
11419                         MLX5_ASSERT(dev_flow->dv.tag_resource);
11420                         sample_act->dr_tag_action =
11421                                 dev_flow->dv.tag_resource->action;
11422                         sample_idx->rix_tag =
11423                                 dev_flow->handle->dvh.rix_tag;
11424                         sample_actions[sample_act->actions_num++] =
11425                                                 sample_act->dr_tag_action;
11426                         /* Recover the mark resource after sample */
11427                         dev_flow->dv.tag_resource = pre_r;
11428                         dev_flow->handle->dvh.rix_tag = pre_rix;
11429                         action_flags |= MLX5_FLOW_ACTION_MARK;
11430                         break;
11431                 }
11432                 case RTE_FLOW_ACTION_TYPE_COUNT:
11433                 {
11434                         if (!flow->counter) {
11435                                 flow->counter =
11436                                         flow_dv_translate_create_counter(dev,
11437                                                 dev_flow, sub_actions->conf,
11438                                                 0);
11439                                 if (!flow->counter)
11440                                         return rte_flow_error_set
11441                                                 (error, rte_errno,
11442                                                 RTE_FLOW_ERROR_TYPE_ACTION,
11443                                                 NULL,
11444                                                 "cannot create counter"
11445                                                 " object.");
11446                         }
11447                         sample_act->dr_cnt_action =
11448                                   (flow_dv_counter_get_by_idx(dev,
11449                                   flow->counter, NULL))->action;
11450                         sample_actions[sample_act->actions_num++] =
11451                                                 sample_act->dr_cnt_action;
11452                         action_flags |= MLX5_FLOW_ACTION_COUNT;
11453                         break;
11454                 }
11455                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
11456                 {
11457                         struct mlx5_flow_dv_port_id_action_resource
11458                                         port_id_resource;
11459                         uint32_t port_id = 0;
11460
11461                         memset(&port_id_resource, 0, sizeof(port_id_resource));
11462                         /* Save the port id resource before sample */
11463                         pre_rix = dev_flow->handle->rix_port_id_action;
11464                         pre_r = dev_flow->dv.port_id_action;
11465                         if (flow_dv_translate_action_port_id(dev, sub_actions,
11466                                                              &port_id, error))
11467                                 return -rte_errno;
11468                         port_id_resource.port_id = port_id;
11469                         if (flow_dv_port_id_action_resource_register
11470                             (dev, &port_id_resource, dev_flow, error))
11471                                 return -rte_errno;
11472                         sample_act->dr_port_id_action =
11473                                 dev_flow->dv.port_id_action->action;
11474                         sample_idx->rix_port_id_action =
11475                                 dev_flow->handle->rix_port_id_action;
11476                         sample_actions[sample_act->actions_num++] =
11477                                                 sample_act->dr_port_id_action;
11478                         /* Recover the port id resource after sample */
11479                         dev_flow->dv.port_id_action = pre_r;
11480                         dev_flow->handle->rix_port_id_action = pre_rix;
11481                         (*num_of_dest)++;
11482                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
11483                         break;
11484                 }
11485                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
11486                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
11487                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
11488                         /* Save the encap resource before sample */
11489                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
11490                         pre_r = dev_flow->dv.encap_decap;
11491                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
11492                                                            dev_flow,
11493                                                            attr->transfer,
11494                                                            error))
11495                                 return -rte_errno;
11496                         sample_act->dr_encap_action =
11497                                 dev_flow->dv.encap_decap->action;
11498                         sample_idx->rix_encap_decap =
11499                                 dev_flow->handle->dvh.rix_encap_decap;
11500                         sample_actions[sample_act->actions_num++] =
11501                                                 sample_act->dr_encap_action;
11502                         /* Recover the encap resource after sample */
11503                         dev_flow->dv.encap_decap = pre_r;
11504                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
11505                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
11506                         break;
11507                 default:
11508                         return rte_flow_error_set(error, EINVAL,
11509                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11510                                 NULL,
11511                                 "Not support for sampler action");
11512                 }
11513         }
11514         sample_act->action_flags = action_flags;
11515         res->ft_id = dev_flow->dv.group;
11516         if (attr->transfer) {
11517                 union {
11518                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
11519                         uint64_t set_action;
11520                 } action_ctx = { .set_action = 0 };
11521
11522                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
11523                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
11524                          MLX5_MODIFICATION_TYPE_SET);
11525                 MLX5_SET(set_action_in, action_ctx.action_in, field,
11526                          MLX5_MODI_META_REG_C_0);
11527                 MLX5_SET(set_action_in, action_ctx.action_in, data,
11528                          priv->vport_meta_tag);
11529                 res->set_action = action_ctx.set_action;
11530         } else if (attr->ingress) {
11531                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
11532         } else {
11533                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
11534         }
11535         return 0;
11536 }
11537
11538 /**
11539  * Convert Sample action to DV specification.
11540  *
11541  * @param[in] dev
11542  *   Pointer to rte_eth_dev structure.
11543  * @param[in, out] dev_flow
11544  *   Pointer to the mlx5_flow.
11545  * @param[in] num_of_dest
11546  *   The num of destination.
11547  * @param[in, out] res
11548  *   Pointer to sample resource.
11549  * @param[in, out] mdest_res
11550  *   Pointer to destination array resource.
11551  * @param[in] sample_actions
11552  *   Pointer to sample path actions list.
11553  * @param[in] action_flags
11554  *   Holds the actions detected until now.
11555  * @param[out] error
11556  *   Pointer to the error structure.
11557  *
11558  * @return
11559  *   0 on success, a negative errno value otherwise and rte_errno is set.
11560  */
11561 static int
11562 flow_dv_create_action_sample(struct rte_eth_dev *dev,
11563                              struct mlx5_flow *dev_flow,
11564                              uint32_t num_of_dest,
11565                              struct mlx5_flow_dv_sample_resource *res,
11566                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
11567                              void **sample_actions,
11568                              uint64_t action_flags,
11569                              struct rte_flow_error *error)
11570 {
11571         /* update normal path action resource into last index of array */
11572         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
11573         struct mlx5_flow_sub_actions_list *sample_act =
11574                                         &mdest_res->sample_act[dest_index];
11575         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11576         struct mlx5_flow_rss_desc *rss_desc;
11577         uint32_t normal_idx = 0;
11578         struct mlx5_hrxq *hrxq;
11579         uint32_t hrxq_idx;
11580
11581         MLX5_ASSERT(wks);
11582         rss_desc = &wks->rss_desc;
11583         if (num_of_dest > 1) {
11584                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
11585                         /* Handle QP action for mirroring */
11586                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11587                                                     rss_desc, &hrxq_idx);
11588                         if (!hrxq)
11589                                 return rte_flow_error_set
11590                                      (error, rte_errno,
11591                                       RTE_FLOW_ERROR_TYPE_ACTION,
11592                                       NULL,
11593                                       "cannot create rx queue");
11594                         normal_idx++;
11595                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
11596                         sample_act->dr_queue_action = hrxq->action;
11597                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11598                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11599                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
11600                 }
11601                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
11602                         normal_idx++;
11603                         mdest_res->sample_idx[dest_index].rix_encap_decap =
11604                                 dev_flow->handle->dvh.rix_encap_decap;
11605                         sample_act->dr_encap_action =
11606                                 dev_flow->dv.encap_decap->action;
11607                         dev_flow->handle->dvh.rix_encap_decap = 0;
11608                 }
11609                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
11610                         normal_idx++;
11611                         mdest_res->sample_idx[dest_index].rix_port_id_action =
11612                                 dev_flow->handle->rix_port_id_action;
11613                         sample_act->dr_port_id_action =
11614                                 dev_flow->dv.port_id_action->action;
11615                         dev_flow->handle->rix_port_id_action = 0;
11616                 }
11617                 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) {
11618                         normal_idx++;
11619                         mdest_res->sample_idx[dest_index].rix_jump =
11620                                 dev_flow->handle->rix_jump;
11621                         sample_act->dr_jump_action =
11622                                 dev_flow->dv.jump->action;
11623                         dev_flow->handle->rix_jump = 0;
11624                 }
11625                 sample_act->actions_num = normal_idx;
11626                 /* update sample action resource into first index of array */
11627                 mdest_res->ft_type = res->ft_type;
11628                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
11629                                 sizeof(struct mlx5_flow_sub_actions_idx));
11630                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
11631                                 sizeof(struct mlx5_flow_sub_actions_list));
11632                 mdest_res->num_of_dest = num_of_dest;
11633                 if (flow_dv_dest_array_resource_register(dev, mdest_res,
11634                                                          dev_flow, error))
11635                         return rte_flow_error_set(error, EINVAL,
11636                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11637                                                   NULL, "can't create sample "
11638                                                   "action");
11639         } else {
11640                 res->sub_actions = sample_actions;
11641                 if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
11642                         return rte_flow_error_set(error, EINVAL,
11643                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11644                                                   NULL,
11645                                                   "can't create sample action");
11646         }
11647         return 0;
11648 }
11649
11650 /**
11651  * Remove an ASO age action from age actions list.
11652  *
11653  * @param[in] dev
11654  *   Pointer to the Ethernet device structure.
11655  * @param[in] age
11656  *   Pointer to the aso age action handler.
11657  */
11658 static void
11659 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev,
11660                                 struct mlx5_aso_age_action *age)
11661 {
11662         struct mlx5_age_info *age_info;
11663         struct mlx5_age_param *age_param = &age->age_params;
11664         struct mlx5_priv *priv = dev->data->dev_private;
11665         uint16_t expected = AGE_CANDIDATE;
11666
11667         age_info = GET_PORT_AGE_INFO(priv);
11668         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
11669                                          AGE_FREE, false, __ATOMIC_RELAXED,
11670                                          __ATOMIC_RELAXED)) {
11671                 /**
11672                  * We need the lock even it is age timeout,
11673                  * since age action may still in process.
11674                  */
11675                 rte_spinlock_lock(&age_info->aged_sl);
11676                 LIST_REMOVE(age, next);
11677                 rte_spinlock_unlock(&age_info->aged_sl);
11678                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
11679         }
11680 }
11681
11682 /**
11683  * Release an ASO age action.
11684  *
11685  * @param[in] dev
11686  *   Pointer to the Ethernet device structure.
11687  * @param[in] age_idx
11688  *   Index of ASO age action to release.
11689  * @param[in] flow
11690  *   True if the release operation is during flow destroy operation.
11691  *   False if the release operation is during action destroy operation.
11692  *
11693  * @return
11694  *   0 when age action was removed, otherwise the number of references.
11695  */
11696 static int
11697 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
11698 {
11699         struct mlx5_priv *priv = dev->data->dev_private;
11700         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11701         struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx);
11702         uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED);
11703
11704         if (!ret) {
11705                 flow_dv_aso_age_remove_from_age(dev, age);
11706                 rte_spinlock_lock(&mng->free_sl);
11707                 LIST_INSERT_HEAD(&mng->free, age, next);
11708                 rte_spinlock_unlock(&mng->free_sl);
11709         }
11710         return ret;
11711 }
11712
11713 /**
11714  * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
11715  *
11716  * @param[in] dev
11717  *   Pointer to the Ethernet device structure.
11718  *
11719  * @return
11720  *   0 on success, otherwise negative errno value and rte_errno is set.
11721  */
11722 static int
11723 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
11724 {
11725         struct mlx5_priv *priv = dev->data->dev_private;
11726         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11727         void *old_pools = mng->pools;
11728         uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
11729         uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
11730         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
11731
11732         if (!pools) {
11733                 rte_errno = ENOMEM;
11734                 return -ENOMEM;
11735         }
11736         if (old_pools) {
11737                 memcpy(pools, old_pools,
11738                        mng->n * sizeof(struct mlx5_flow_counter_pool *));
11739                 mlx5_free(old_pools);
11740         } else {
11741                 /* First ASO flow hit allocation - starting ASO data-path. */
11742                 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh);
11743
11744                 if (ret) {
11745                         mlx5_free(pools);
11746                         return ret;
11747                 }
11748         }
11749         mng->n = resize;
11750         mng->pools = pools;
11751         return 0;
11752 }
11753
11754 /**
11755  * Create and initialize a new ASO aging pool.
11756  *
11757  * @param[in] dev
11758  *   Pointer to the Ethernet device structure.
11759  * @param[out] age_free
11760  *   Where to put the pointer of a new age action.
11761  *
11762  * @return
11763  *   The age actions pool pointer and @p age_free is set on success,
11764  *   NULL otherwise and rte_errno is set.
11765  */
11766 static struct mlx5_aso_age_pool *
11767 flow_dv_age_pool_create(struct rte_eth_dev *dev,
11768                         struct mlx5_aso_age_action **age_free)
11769 {
11770         struct mlx5_priv *priv = dev->data->dev_private;
11771         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11772         struct mlx5_aso_age_pool *pool = NULL;
11773         struct mlx5_devx_obj *obj = NULL;
11774         uint32_t i;
11775
11776         obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->ctx,
11777                                                     priv->sh->pdn);
11778         if (!obj) {
11779                 rte_errno = ENODATA;
11780                 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX.");
11781                 return NULL;
11782         }
11783         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
11784         if (!pool) {
11785                 claim_zero(mlx5_devx_cmd_destroy(obj));
11786                 rte_errno = ENOMEM;
11787                 return NULL;
11788         }
11789         pool->flow_hit_aso_obj = obj;
11790         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
11791         rte_spinlock_lock(&mng->resize_sl);
11792         pool->index = mng->next;
11793         /* Resize pools array if there is no room for the new pool in it. */
11794         if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) {
11795                 claim_zero(mlx5_devx_cmd_destroy(obj));
11796                 mlx5_free(pool);
11797                 rte_spinlock_unlock(&mng->resize_sl);
11798                 return NULL;
11799         }
11800         mng->pools[pool->index] = pool;
11801         mng->next++;
11802         rte_spinlock_unlock(&mng->resize_sl);
11803         /* Assign the first action in the new pool, the rest go to free list. */
11804         *age_free = &pool->actions[0];
11805         for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) {
11806                 pool->actions[i].offset = i;
11807                 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next);
11808         }
11809         return pool;
11810 }
11811
11812 /**
11813  * Allocate a ASO aging bit.
11814  *
11815  * @param[in] dev
11816  *   Pointer to the Ethernet device structure.
11817  * @param[out] error
11818  *   Pointer to the error structure.
11819  *
11820  * @return
11821  *   Index to ASO age action on success, 0 otherwise and rte_errno is set.
11822  */
11823 static uint32_t
11824 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
11825 {
11826         struct mlx5_priv *priv = dev->data->dev_private;
11827         const struct mlx5_aso_age_pool *pool;
11828         struct mlx5_aso_age_action *age_free = NULL;
11829         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11830
11831         MLX5_ASSERT(mng);
11832         /* Try to get the next free age action bit. */
11833         rte_spinlock_lock(&mng->free_sl);
11834         age_free = LIST_FIRST(&mng->free);
11835         if (age_free) {
11836                 LIST_REMOVE(age_free, next);
11837         } else if (!flow_dv_age_pool_create(dev, &age_free)) {
11838                 rte_spinlock_unlock(&mng->free_sl);
11839                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
11840                                    NULL, "failed to create ASO age pool");
11841                 return 0; /* 0 is an error. */
11842         }
11843         rte_spinlock_unlock(&mng->free_sl);
11844         pool = container_of
11845           ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL])
11846                   (age_free - age_free->offset), const struct mlx5_aso_age_pool,
11847                                                                        actions);
11848         if (!age_free->dr_action) {
11849                 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0,
11850                                                  error);
11851
11852                 if (reg_c < 0) {
11853                         rte_flow_error_set(error, rte_errno,
11854                                            RTE_FLOW_ERROR_TYPE_ACTION,
11855                                            NULL, "failed to get reg_c "
11856                                            "for ASO flow hit");
11857                         return 0; /* 0 is an error. */
11858                 }
11859 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
11860                 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso
11861                                 (priv->sh->rx_domain,
11862                                  pool->flow_hit_aso_obj->obj, age_free->offset,
11863                                  MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET,
11864                                  (reg_c - REG_C_0));
11865 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
11866                 if (!age_free->dr_action) {
11867                         rte_errno = errno;
11868                         rte_spinlock_lock(&mng->free_sl);
11869                         LIST_INSERT_HEAD(&mng->free, age_free, next);
11870                         rte_spinlock_unlock(&mng->free_sl);
11871                         rte_flow_error_set(error, rte_errno,
11872                                            RTE_FLOW_ERROR_TYPE_ACTION,
11873                                            NULL, "failed to create ASO "
11874                                            "flow hit action");
11875                         return 0; /* 0 is an error. */
11876                 }
11877         }
11878         __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED);
11879         return pool->index | ((age_free->offset + 1) << 16);
11880 }
11881
11882 /**
11883  * Initialize flow ASO age parameters.
11884  *
11885  * @param[in] dev
11886  *   Pointer to rte_eth_dev structure.
11887  * @param[in] age_idx
11888  *   Index of ASO age action.
11889  * @param[in] context
11890  *   Pointer to flow counter age context.
11891  * @param[in] timeout
11892  *   Aging timeout in seconds.
11893  *
11894  */
11895 static void
11896 flow_dv_aso_age_params_init(struct rte_eth_dev *dev,
11897                             uint32_t age_idx,
11898                             void *context,
11899                             uint32_t timeout)
11900 {
11901         struct mlx5_aso_age_action *aso_age;
11902
11903         aso_age = flow_aso_age_get_by_idx(dev, age_idx);
11904         MLX5_ASSERT(aso_age);
11905         aso_age->age_params.context = context;
11906         aso_age->age_params.timeout = timeout;
11907         aso_age->age_params.port_id = dev->data->port_id;
11908         __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0,
11909                          __ATOMIC_RELAXED);
11910         __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE,
11911                          __ATOMIC_RELAXED);
11912 }
11913
11914 static void
11915 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask,
11916                                const struct rte_flow_item_integrity *value,
11917                                void *headers_m, void *headers_v)
11918 {
11919         if (mask->l4_ok) {
11920                 /* application l4_ok filter aggregates all hardware l4 filters
11921                  * therefore hw l4_checksum_ok must be implicitly added here.
11922                  */
11923                 struct rte_flow_item_integrity local_item;
11924
11925                 local_item.l4_csum_ok = 1;
11926                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
11927                          local_item.l4_csum_ok);
11928                 if (value->l4_ok) {
11929                         /* application l4_ok = 1 matches sets both hw flags
11930                          * l4_ok and l4_checksum_ok flags to 1.
11931                          */
11932                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11933                                  l4_checksum_ok, local_item.l4_csum_ok);
11934                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok,
11935                                  mask->l4_ok);
11936                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok,
11937                                  value->l4_ok);
11938                 } else {
11939                         /* application l4_ok = 0 matches on hw flag
11940                          * l4_checksum_ok = 0 only.
11941                          */
11942                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11943                                  l4_checksum_ok, 0);
11944                 }
11945         } else if (mask->l4_csum_ok) {
11946                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
11947                          mask->l4_csum_ok);
11948                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
11949                          value->l4_csum_ok);
11950         }
11951 }
11952
11953 static void
11954 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask,
11955                                const struct rte_flow_item_integrity *value,
11956                                void *headers_m, void *headers_v,
11957                                bool is_ipv4)
11958 {
11959         if (mask->l3_ok) {
11960                 /* application l3_ok filter aggregates all hardware l3 filters
11961                  * therefore hw ipv4_checksum_ok must be implicitly added here.
11962                  */
11963                 struct rte_flow_item_integrity local_item;
11964
11965                 local_item.ipv4_csum_ok = !!is_ipv4;
11966                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
11967                          local_item.ipv4_csum_ok);
11968                 if (value->l3_ok) {
11969                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11970                                  ipv4_checksum_ok, local_item.ipv4_csum_ok);
11971                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok,
11972                                  mask->l3_ok);
11973                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok,
11974                                  value->l3_ok);
11975                 } else {
11976                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11977                                  ipv4_checksum_ok, 0);
11978                 }
11979         } else if (mask->ipv4_csum_ok) {
11980                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
11981                          mask->ipv4_csum_ok);
11982                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
11983                          value->ipv4_csum_ok);
11984         }
11985 }
11986
11987 static void
11988 flow_dv_translate_item_integrity(void *matcher, void *key,
11989                                  const struct rte_flow_item *head_item,
11990                                  const struct rte_flow_item *integrity_item)
11991 {
11992         const struct rte_flow_item_integrity *mask = integrity_item->mask;
11993         const struct rte_flow_item_integrity *value = integrity_item->spec;
11994         const struct rte_flow_item *tunnel_item, *end_item, *item;
11995         void *headers_m;
11996         void *headers_v;
11997         uint32_t l3_protocol;
11998
11999         if (!value)
12000                 return;
12001         if (!mask)
12002                 mask = &rte_flow_item_integrity_mask;
12003         if (value->level > 1) {
12004                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12005                                          inner_headers);
12006                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
12007         } else {
12008                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12009                                          outer_headers);
12010                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
12011         }
12012         tunnel_item = mlx5_flow_find_tunnel_item(head_item);
12013         if (value->level > 1) {
12014                 /* tunnel item was verified during the item validation */
12015                 item = tunnel_item;
12016                 end_item = mlx5_find_end_item(tunnel_item);
12017         } else {
12018                 item = head_item;
12019                 end_item = tunnel_item ? tunnel_item :
12020                            mlx5_find_end_item(integrity_item);
12021         }
12022         l3_protocol = mask->l3_ok ?
12023                       mlx5_flow_locate_proto_l3(&item, end_item) : 0;
12024         flow_dv_translate_integrity_l3(mask, value, headers_m, headers_v,
12025                                        l3_protocol == RTE_ETHER_TYPE_IPV4);
12026         flow_dv_translate_integrity_l4(mask, value, headers_m, headers_v);
12027 }
12028
12029 /**
12030  * Prepares DV flow counter with aging configuration.
12031  * Gets it by index when exists, creates a new one when doesn't.
12032  *
12033  * @param[in] dev
12034  *   Pointer to rte_eth_dev structure.
12035  * @param[in] dev_flow
12036  *   Pointer to the mlx5_flow.
12037  * @param[in, out] flow
12038  *   Pointer to the sub flow.
12039  * @param[in] count
12040  *   Pointer to the counter action configuration.
12041  * @param[in] age
12042  *   Pointer to the aging action configuration.
12043  * @param[out] error
12044  *   Pointer to the error structure.
12045  *
12046  * @return
12047  *   Pointer to the counter, NULL otherwise.
12048  */
12049 static struct mlx5_flow_counter *
12050 flow_dv_prepare_counter(struct rte_eth_dev *dev,
12051                         struct mlx5_flow *dev_flow,
12052                         struct rte_flow *flow,
12053                         const struct rte_flow_action_count *count,
12054                         const struct rte_flow_action_age *age,
12055                         struct rte_flow_error *error)
12056 {
12057         if (!flow->counter) {
12058                 flow->counter = flow_dv_translate_create_counter(dev, dev_flow,
12059                                                                  count, age);
12060                 if (!flow->counter) {
12061                         rte_flow_error_set(error, rte_errno,
12062                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12063                                            "cannot create counter object.");
12064                         return NULL;
12065                 }
12066         }
12067         return flow_dv_counter_get_by_idx(dev, flow->counter, NULL);
12068 }
12069
12070 /*
12071  * Release an ASO CT action by its own device.
12072  *
12073  * @param[in] dev
12074  *   Pointer to the Ethernet device structure.
12075  * @param[in] idx
12076  *   Index of ASO CT action to release.
12077  *
12078  * @return
12079  *   0 when CT action was removed, otherwise the number of references.
12080  */
12081 static inline int
12082 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx)
12083 {
12084         struct mlx5_priv *priv = dev->data->dev_private;
12085         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12086         uint32_t ret;
12087         struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12088         enum mlx5_aso_ct_state state =
12089                         __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
12090
12091         /* Cannot release when CT is in the ASO SQ. */
12092         if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY)
12093                 return -1;
12094         ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED);
12095         if (!ret) {
12096                 if (ct->dr_action_orig) {
12097 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12098                         claim_zero(mlx5_glue->destroy_flow_action
12099                                         (ct->dr_action_orig));
12100 #endif
12101                         ct->dr_action_orig = NULL;
12102                 }
12103                 if (ct->dr_action_rply) {
12104 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12105                         claim_zero(mlx5_glue->destroy_flow_action
12106                                         (ct->dr_action_rply));
12107 #endif
12108                         ct->dr_action_rply = NULL;
12109                 }
12110                 /* Clear the state to free, no need in 1st allocation. */
12111                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE);
12112                 rte_spinlock_lock(&mng->ct_sl);
12113                 LIST_INSERT_HEAD(&mng->free_cts, ct, next);
12114                 rte_spinlock_unlock(&mng->ct_sl);
12115         }
12116         return (int)ret;
12117 }
12118
12119 static inline int
12120 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx)
12121 {
12122         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx);
12123         uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx);
12124         struct rte_eth_dev *owndev = &rte_eth_devices[owner];
12125         RTE_SET_USED(dev);
12126
12127         MLX5_ASSERT(owner < RTE_MAX_ETHPORTS);
12128         if (dev->data->dev_started != 1)
12129                 return -1;
12130         return flow_dv_aso_ct_dev_release(owndev, idx);
12131 }
12132
12133 /*
12134  * Resize the ASO CT pools array by 64 pools.
12135  *
12136  * @param[in] dev
12137  *   Pointer to the Ethernet device structure.
12138  *
12139  * @return
12140  *   0 on success, otherwise negative errno value and rte_errno is set.
12141  */
12142 static int
12143 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev)
12144 {
12145         struct mlx5_priv *priv = dev->data->dev_private;
12146         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12147         void *old_pools = mng->pools;
12148         /* Magic number now, need a macro. */
12149         uint32_t resize = mng->n + 64;
12150         uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize;
12151         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
12152
12153         if (!pools) {
12154                 rte_errno = ENOMEM;
12155                 return -rte_errno;
12156         }
12157         rte_rwlock_write_lock(&mng->resize_rwl);
12158         /* ASO SQ/QP was already initialized in the startup. */
12159         if (old_pools) {
12160                 /* Realloc could be an alternative choice. */
12161                 rte_memcpy(pools, old_pools,
12162                            mng->n * sizeof(struct mlx5_aso_ct_pool *));
12163                 mlx5_free(old_pools);
12164         }
12165         mng->n = resize;
12166         mng->pools = pools;
12167         rte_rwlock_write_unlock(&mng->resize_rwl);
12168         return 0;
12169 }
12170
12171 /*
12172  * Create and initialize a new ASO CT pool.
12173  *
12174  * @param[in] dev
12175  *   Pointer to the Ethernet device structure.
12176  * @param[out] ct_free
12177  *   Where to put the pointer of a new CT action.
12178  *
12179  * @return
12180  *   The CT actions pool pointer and @p ct_free is set on success,
12181  *   NULL otherwise and rte_errno is set.
12182  */
12183 static struct mlx5_aso_ct_pool *
12184 flow_dv_ct_pool_create(struct rte_eth_dev *dev,
12185                        struct mlx5_aso_ct_action **ct_free)
12186 {
12187         struct mlx5_priv *priv = dev->data->dev_private;
12188         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12189         struct mlx5_aso_ct_pool *pool = NULL;
12190         struct mlx5_devx_obj *obj = NULL;
12191         uint32_t i;
12192         uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL);
12193
12194         obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->ctx,
12195                                                 priv->sh->pdn, log_obj_size);
12196         if (!obj) {
12197                 rte_errno = ENODATA;
12198                 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
12199                 return NULL;
12200         }
12201         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
12202         if (!pool) {
12203                 rte_errno = ENOMEM;
12204                 claim_zero(mlx5_devx_cmd_destroy(obj));
12205                 return NULL;
12206         }
12207         pool->devx_obj = obj;
12208         pool->index = mng->next;
12209         /* Resize pools array if there is no room for the new pool in it. */
12210         if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) {
12211                 claim_zero(mlx5_devx_cmd_destroy(obj));
12212                 mlx5_free(pool);
12213                 return NULL;
12214         }
12215         mng->pools[pool->index] = pool;
12216         mng->next++;
12217         /* Assign the first action in the new pool, the rest go to free list. */
12218         *ct_free = &pool->actions[0];
12219         /* Lock outside, the list operation is safe here. */
12220         for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) {
12221                 /* refcnt is 0 when allocating the memory. */
12222                 pool->actions[i].offset = i;
12223                 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next);
12224         }
12225         return pool;
12226 }
12227
12228 /*
12229  * Allocate a ASO CT action from free list.
12230  *
12231  * @param[in] dev
12232  *   Pointer to the Ethernet device structure.
12233  * @param[out] error
12234  *   Pointer to the error structure.
12235  *
12236  * @return
12237  *   Index to ASO CT action on success, 0 otherwise and rte_errno is set.
12238  */
12239 static uint32_t
12240 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12241 {
12242         struct mlx5_priv *priv = dev->data->dev_private;
12243         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12244         struct mlx5_aso_ct_action *ct = NULL;
12245         struct mlx5_aso_ct_pool *pool;
12246         uint8_t reg_c;
12247         uint32_t ct_idx;
12248
12249         MLX5_ASSERT(mng);
12250         if (!priv->config.devx) {
12251                 rte_errno = ENOTSUP;
12252                 return 0;
12253         }
12254         /* Get a free CT action, if no, a new pool will be created. */
12255         rte_spinlock_lock(&mng->ct_sl);
12256         ct = LIST_FIRST(&mng->free_cts);
12257         if (ct) {
12258                 LIST_REMOVE(ct, next);
12259         } else if (!flow_dv_ct_pool_create(dev, &ct)) {
12260                 rte_spinlock_unlock(&mng->ct_sl);
12261                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12262                                    NULL, "failed to create ASO CT pool");
12263                 return 0;
12264         }
12265         rte_spinlock_unlock(&mng->ct_sl);
12266         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
12267         ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset);
12268         /* 0: inactive, 1: created, 2+: used by flows. */
12269         __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED);
12270         reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error);
12271         if (!ct->dr_action_orig) {
12272 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12273                 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso
12274                         (priv->sh->rx_domain, pool->devx_obj->obj,
12275                          ct->offset,
12276                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR,
12277                          reg_c - REG_C_0);
12278 #else
12279                 RTE_SET_USED(reg_c);
12280 #endif
12281                 if (!ct->dr_action_orig) {
12282                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12283                         rte_flow_error_set(error, rte_errno,
12284                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12285                                            "failed to create ASO CT action");
12286                         return 0;
12287                 }
12288         }
12289         if (!ct->dr_action_rply) {
12290 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12291                 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso
12292                         (priv->sh->rx_domain, pool->devx_obj->obj,
12293                          ct->offset,
12294                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER,
12295                          reg_c - REG_C_0);
12296 #endif
12297                 if (!ct->dr_action_rply) {
12298                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12299                         rte_flow_error_set(error, rte_errno,
12300                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12301                                            "failed to create ASO CT action");
12302                         return 0;
12303                 }
12304         }
12305         return ct_idx;
12306 }
12307
12308 /*
12309  * Create a conntrack object with context and actions by using ASO mechanism.
12310  *
12311  * @param[in] dev
12312  *   Pointer to rte_eth_dev structure.
12313  * @param[in] pro
12314  *   Pointer to conntrack information profile.
12315  * @param[out] error
12316  *   Pointer to the error structure.
12317  *
12318  * @return
12319  *   Index to conntrack object on success, 0 otherwise.
12320  */
12321 static uint32_t
12322 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev,
12323                                    const struct rte_flow_action_conntrack *pro,
12324                                    struct rte_flow_error *error)
12325 {
12326         struct mlx5_priv *priv = dev->data->dev_private;
12327         struct mlx5_dev_ctx_shared *sh = priv->sh;
12328         struct mlx5_aso_ct_action *ct;
12329         uint32_t idx;
12330
12331         if (!sh->ct_aso_en)
12332                 return rte_flow_error_set(error, ENOTSUP,
12333                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12334                                           "Connection is not supported");
12335         idx = flow_dv_aso_ct_alloc(dev, error);
12336         if (!idx)
12337                 return rte_flow_error_set(error, rte_errno,
12338                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12339                                           "Failed to allocate CT object");
12340         ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12341         if (mlx5_aso_ct_update_by_wqe(sh, ct, pro))
12342                 return rte_flow_error_set(error, EBUSY,
12343                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12344                                           "Failed to update CT");
12345         ct->is_original = !!pro->is_original_dir;
12346         ct->peer = pro->peer_port;
12347         return idx;
12348 }
12349
12350 /**
12351  * Fill the flow with DV spec, lock free
12352  * (mutex should be acquired by caller).
12353  *
12354  * @param[in] dev
12355  *   Pointer to rte_eth_dev structure.
12356  * @param[in, out] dev_flow
12357  *   Pointer to the sub flow.
12358  * @param[in] attr
12359  *   Pointer to the flow attributes.
12360  * @param[in] items
12361  *   Pointer to the list of items.
12362  * @param[in] actions
12363  *   Pointer to the list of actions.
12364  * @param[out] error
12365  *   Pointer to the error structure.
12366  *
12367  * @return
12368  *   0 on success, a negative errno value otherwise and rte_errno is set.
12369  */
12370 static int
12371 flow_dv_translate(struct rte_eth_dev *dev,
12372                   struct mlx5_flow *dev_flow,
12373                   const struct rte_flow_attr *attr,
12374                   const struct rte_flow_item items[],
12375                   const struct rte_flow_action actions[],
12376                   struct rte_flow_error *error)
12377 {
12378         struct mlx5_priv *priv = dev->data->dev_private;
12379         struct mlx5_dev_config *dev_conf = &priv->config;
12380         struct rte_flow *flow = dev_flow->flow;
12381         struct mlx5_flow_handle *handle = dev_flow->handle;
12382         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12383         struct mlx5_flow_rss_desc *rss_desc;
12384         uint64_t item_flags = 0;
12385         uint64_t last_item = 0;
12386         uint64_t action_flags = 0;
12387         struct mlx5_flow_dv_matcher matcher = {
12388                 .mask = {
12389                         .size = sizeof(matcher.mask.buf),
12390                 },
12391         };
12392         int actions_n = 0;
12393         bool actions_end = false;
12394         union {
12395                 struct mlx5_flow_dv_modify_hdr_resource res;
12396                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
12397                             sizeof(struct mlx5_modification_cmd) *
12398                             (MLX5_MAX_MODIFY_NUM + 1)];
12399         } mhdr_dummy;
12400         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
12401         const struct rte_flow_action_count *count = NULL;
12402         const struct rte_flow_action_age *non_shared_age = NULL;
12403         union flow_dv_attr flow_attr = { .attr = 0 };
12404         uint32_t tag_be;
12405         union mlx5_flow_tbl_key tbl_key;
12406         uint32_t modify_action_position = UINT32_MAX;
12407         void *match_mask = matcher.mask.buf;
12408         void *match_value = dev_flow->dv.value.buf;
12409         uint8_t next_protocol = 0xff;
12410         struct rte_vlan_hdr vlan = { 0 };
12411         struct mlx5_flow_dv_dest_array_resource mdest_res;
12412         struct mlx5_flow_dv_sample_resource sample_res;
12413         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
12414         const struct rte_flow_action_sample *sample = NULL;
12415         struct mlx5_flow_sub_actions_list *sample_act;
12416         uint32_t sample_act_pos = UINT32_MAX;
12417         uint32_t age_act_pos = UINT32_MAX;
12418         uint32_t num_of_dest = 0;
12419         int tmp_actions_n = 0;
12420         uint32_t table;
12421         int ret = 0;
12422         const struct mlx5_flow_tunnel *tunnel = NULL;
12423         struct flow_grp_info grp_info = {
12424                 .external = !!dev_flow->external,
12425                 .transfer = !!attr->transfer,
12426                 .fdb_def_rule = !!priv->fdb_def_rule,
12427                 .skip_scale = dev_flow->skip_scale &
12428                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
12429                 .std_tbl_fix = true,
12430         };
12431         const struct rte_flow_item *head_item = items;
12432
12433         if (!wks)
12434                 return rte_flow_error_set(error, ENOMEM,
12435                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12436                                           NULL,
12437                                           "failed to push flow workspace");
12438         rss_desc = &wks->rss_desc;
12439         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
12440         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
12441         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12442                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12443         /* update normal path action resource into last index of array */
12444         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
12445         if (is_tunnel_offload_active(dev)) {
12446                 if (dev_flow->tunnel) {
12447                         RTE_VERIFY(dev_flow->tof_type ==
12448                                    MLX5_TUNNEL_OFFLOAD_MISS_RULE);
12449                         tunnel = dev_flow->tunnel;
12450                 } else {
12451                         tunnel = mlx5_get_tof(items, actions,
12452                                               &dev_flow->tof_type);
12453                         dev_flow->tunnel = tunnel;
12454                 }
12455                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
12456                                         (dev, attr, tunnel, dev_flow->tof_type);
12457         }
12458         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12459                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12460         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
12461                                        &grp_info, error);
12462         if (ret)
12463                 return ret;
12464         dev_flow->dv.group = table;
12465         if (attr->transfer)
12466                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12467         /* number of actions must be set to 0 in case of dirty stack. */
12468         mhdr_res->actions_num = 0;
12469         if (is_flow_tunnel_match_rule(dev_flow->tof_type)) {
12470                 /*
12471                  * do not add decap action if match rule drops packet
12472                  * HW rejects rules with decap & drop
12473                  *
12474                  * if tunnel match rule was inserted before matching tunnel set
12475                  * rule flow table used in the match rule must be registered.
12476                  * current implementation handles that in the
12477                  * flow_dv_match_register() at the function end.
12478                  */
12479                 bool add_decap = true;
12480                 const struct rte_flow_action *ptr = actions;
12481
12482                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
12483                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
12484                                 add_decap = false;
12485                                 break;
12486                         }
12487                 }
12488                 if (add_decap) {
12489                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12490                                                            attr->transfer,
12491                                                            error))
12492                                 return -rte_errno;
12493                         dev_flow->dv.actions[actions_n++] =
12494                                         dev_flow->dv.encap_decap->action;
12495                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12496                 }
12497         }
12498         for (; !actions_end ; actions++) {
12499                 const struct rte_flow_action_queue *queue;
12500                 const struct rte_flow_action_rss *rss;
12501                 const struct rte_flow_action *action = actions;
12502                 const uint8_t *rss_key;
12503                 struct mlx5_flow_tbl_resource *tbl;
12504                 struct mlx5_aso_age_action *age_act;
12505                 struct mlx5_flow_counter *cnt_act;
12506                 uint32_t port_id = 0;
12507                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
12508                 int action_type = actions->type;
12509                 const struct rte_flow_action *found_action = NULL;
12510                 uint32_t jump_group = 0;
12511                 uint32_t owner_idx;
12512                 struct mlx5_aso_ct_action *ct;
12513
12514                 if (!mlx5_flow_os_action_supported(action_type))
12515                         return rte_flow_error_set(error, ENOTSUP,
12516                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12517                                                   actions,
12518                                                   "action not supported");
12519                 switch (action_type) {
12520                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
12521                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
12522                         break;
12523                 case RTE_FLOW_ACTION_TYPE_VOID:
12524                         break;
12525                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
12526                         if (flow_dv_translate_action_port_id(dev, action,
12527                                                              &port_id, error))
12528                                 return -rte_errno;
12529                         port_id_resource.port_id = port_id;
12530                         MLX5_ASSERT(!handle->rix_port_id_action);
12531                         if (flow_dv_port_id_action_resource_register
12532                             (dev, &port_id_resource, dev_flow, error))
12533                                 return -rte_errno;
12534                         dev_flow->dv.actions[actions_n++] =
12535                                         dev_flow->dv.port_id_action->action;
12536                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12537                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
12538                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12539                         num_of_dest++;
12540                         break;
12541                 case RTE_FLOW_ACTION_TYPE_FLAG:
12542                         action_flags |= MLX5_FLOW_ACTION_FLAG;
12543                         dev_flow->handle->mark = 1;
12544                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12545                                 struct rte_flow_action_mark mark = {
12546                                         .id = MLX5_FLOW_MARK_DEFAULT,
12547                                 };
12548
12549                                 if (flow_dv_convert_action_mark(dev, &mark,
12550                                                                 mhdr_res,
12551                                                                 error))
12552                                         return -rte_errno;
12553                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12554                                 break;
12555                         }
12556                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
12557                         /*
12558                          * Only one FLAG or MARK is supported per device flow
12559                          * right now. So the pointer to the tag resource must be
12560                          * zero before the register process.
12561                          */
12562                         MLX5_ASSERT(!handle->dvh.rix_tag);
12563                         if (flow_dv_tag_resource_register(dev, tag_be,
12564                                                           dev_flow, error))
12565                                 return -rte_errno;
12566                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12567                         dev_flow->dv.actions[actions_n++] =
12568                                         dev_flow->dv.tag_resource->action;
12569                         break;
12570                 case RTE_FLOW_ACTION_TYPE_MARK:
12571                         action_flags |= MLX5_FLOW_ACTION_MARK;
12572                         dev_flow->handle->mark = 1;
12573                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12574                                 const struct rte_flow_action_mark *mark =
12575                                         (const struct rte_flow_action_mark *)
12576                                                 actions->conf;
12577
12578                                 if (flow_dv_convert_action_mark(dev, mark,
12579                                                                 mhdr_res,
12580                                                                 error))
12581                                         return -rte_errno;
12582                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12583                                 break;
12584                         }
12585                         /* Fall-through */
12586                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
12587                         /* Legacy (non-extensive) MARK action. */
12588                         tag_be = mlx5_flow_mark_set
12589                               (((const struct rte_flow_action_mark *)
12590                                (actions->conf))->id);
12591                         MLX5_ASSERT(!handle->dvh.rix_tag);
12592                         if (flow_dv_tag_resource_register(dev, tag_be,
12593                                                           dev_flow, error))
12594                                 return -rte_errno;
12595                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12596                         dev_flow->dv.actions[actions_n++] =
12597                                         dev_flow->dv.tag_resource->action;
12598                         break;
12599                 case RTE_FLOW_ACTION_TYPE_SET_META:
12600                         if (flow_dv_convert_action_set_meta
12601                                 (dev, mhdr_res, attr,
12602                                  (const struct rte_flow_action_set_meta *)
12603                                   actions->conf, error))
12604                                 return -rte_errno;
12605                         action_flags |= MLX5_FLOW_ACTION_SET_META;
12606                         break;
12607                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
12608                         if (flow_dv_convert_action_set_tag
12609                                 (dev, mhdr_res,
12610                                  (const struct rte_flow_action_set_tag *)
12611                                   actions->conf, error))
12612                                 return -rte_errno;
12613                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12614                         break;
12615                 case RTE_FLOW_ACTION_TYPE_DROP:
12616                         action_flags |= MLX5_FLOW_ACTION_DROP;
12617                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
12618                         break;
12619                 case RTE_FLOW_ACTION_TYPE_QUEUE:
12620                         queue = actions->conf;
12621                         rss_desc->queue_num = 1;
12622                         rss_desc->queue[0] = queue->index;
12623                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
12624                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
12625                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
12626                         num_of_dest++;
12627                         break;
12628                 case RTE_FLOW_ACTION_TYPE_RSS:
12629                         rss = actions->conf;
12630                         memcpy(rss_desc->queue, rss->queue,
12631                                rss->queue_num * sizeof(uint16_t));
12632                         rss_desc->queue_num = rss->queue_num;
12633                         /* NULL RSS key indicates default RSS key. */
12634                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
12635                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
12636                         /*
12637                          * rss->level and rss.types should be set in advance
12638                          * when expanding items for RSS.
12639                          */
12640                         action_flags |= MLX5_FLOW_ACTION_RSS;
12641                         dev_flow->handle->fate_action = rss_desc->shared_rss ?
12642                                 MLX5_FLOW_FATE_SHARED_RSS :
12643                                 MLX5_FLOW_FATE_QUEUE;
12644                         break;
12645                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
12646                         flow->age = (uint32_t)(uintptr_t)(action->conf);
12647                         age_act = flow_aso_age_get_by_idx(dev, flow->age);
12648                         __atomic_fetch_add(&age_act->refcnt, 1,
12649                                            __ATOMIC_RELAXED);
12650                         age_act_pos = actions_n++;
12651                         action_flags |= MLX5_FLOW_ACTION_AGE;
12652                         break;
12653                 case RTE_FLOW_ACTION_TYPE_AGE:
12654                         non_shared_age = action->conf;
12655                         age_act_pos = actions_n++;
12656                         action_flags |= MLX5_FLOW_ACTION_AGE;
12657                         break;
12658                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
12659                         flow->counter = (uint32_t)(uintptr_t)(action->conf);
12660                         cnt_act = flow_dv_counter_get_by_idx(dev, flow->counter,
12661                                                              NULL);
12662                         __atomic_fetch_add(&cnt_act->shared_info.refcnt, 1,
12663                                            __ATOMIC_RELAXED);
12664                         /* Save information first, will apply later. */
12665                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12666                         break;
12667                 case RTE_FLOW_ACTION_TYPE_COUNT:
12668                         if (!dev_conf->devx) {
12669                                 return rte_flow_error_set
12670                                               (error, ENOTSUP,
12671                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12672                                                NULL,
12673                                                "count action not supported");
12674                         }
12675                         /* Save information first, will apply later. */
12676                         count = action->conf;
12677                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12678                         break;
12679                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
12680                         dev_flow->dv.actions[actions_n++] =
12681                                                 priv->sh->pop_vlan_action;
12682                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
12683                         break;
12684                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
12685                         if (!(action_flags &
12686                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
12687                                 flow_dev_get_vlan_info_from_items(items, &vlan);
12688                         vlan.eth_proto = rte_be_to_cpu_16
12689                              ((((const struct rte_flow_action_of_push_vlan *)
12690                                                    actions->conf)->ethertype));
12691                         found_action = mlx5_flow_find_action
12692                                         (actions + 1,
12693                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
12694                         if (found_action)
12695                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12696                         found_action = mlx5_flow_find_action
12697                                         (actions + 1,
12698                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
12699                         if (found_action)
12700                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12701                         if (flow_dv_create_action_push_vlan
12702                                             (dev, attr, &vlan, dev_flow, error))
12703                                 return -rte_errno;
12704                         dev_flow->dv.actions[actions_n++] =
12705                                         dev_flow->dv.push_vlan_res->action;
12706                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
12707                         break;
12708                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
12709                         /* of_vlan_push action handled this action */
12710                         MLX5_ASSERT(action_flags &
12711                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
12712                         break;
12713                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
12714                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
12715                                 break;
12716                         flow_dev_get_vlan_info_from_items(items, &vlan);
12717                         mlx5_update_vlan_vid_pcp(actions, &vlan);
12718                         /* If no VLAN push - this is a modify header action */
12719                         if (flow_dv_convert_action_modify_vlan_vid
12720                                                 (mhdr_res, actions, error))
12721                                 return -rte_errno;
12722                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
12723                         break;
12724                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
12725                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
12726                         if (flow_dv_create_action_l2_encap(dev, actions,
12727                                                            dev_flow,
12728                                                            attr->transfer,
12729                                                            error))
12730                                 return -rte_errno;
12731                         dev_flow->dv.actions[actions_n++] =
12732                                         dev_flow->dv.encap_decap->action;
12733                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12734                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12735                                 sample_act->action_flags |=
12736                                                         MLX5_FLOW_ACTION_ENCAP;
12737                         break;
12738                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
12739                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
12740                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12741                                                            attr->transfer,
12742                                                            error))
12743                                 return -rte_errno;
12744                         dev_flow->dv.actions[actions_n++] =
12745                                         dev_flow->dv.encap_decap->action;
12746                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12747                         break;
12748                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
12749                         /* Handle encap with preceding decap. */
12750                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
12751                                 if (flow_dv_create_action_raw_encap
12752                                         (dev, actions, dev_flow, attr, error))
12753                                         return -rte_errno;
12754                                 dev_flow->dv.actions[actions_n++] =
12755                                         dev_flow->dv.encap_decap->action;
12756                         } else {
12757                                 /* Handle encap without preceding decap. */
12758                                 if (flow_dv_create_action_l2_encap
12759                                     (dev, actions, dev_flow, attr->transfer,
12760                                      error))
12761                                         return -rte_errno;
12762                                 dev_flow->dv.actions[actions_n++] =
12763                                         dev_flow->dv.encap_decap->action;
12764                         }
12765                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12766                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12767                                 sample_act->action_flags |=
12768                                                         MLX5_FLOW_ACTION_ENCAP;
12769                         break;
12770                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
12771                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
12772                                 ;
12773                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
12774                                 if (flow_dv_create_action_l2_decap
12775                                     (dev, dev_flow, attr->transfer, error))
12776                                         return -rte_errno;
12777                                 dev_flow->dv.actions[actions_n++] =
12778                                         dev_flow->dv.encap_decap->action;
12779                         }
12780                         /* If decap is followed by encap, handle it at encap. */
12781                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12782                         break;
12783                 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP:
12784                         dev_flow->dv.actions[actions_n++] =
12785                                 (void *)(uintptr_t)action->conf;
12786                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12787                         break;
12788                 case RTE_FLOW_ACTION_TYPE_JUMP:
12789                         jump_group = ((const struct rte_flow_action_jump *)
12790                                                         action->conf)->group;
12791                         grp_info.std_tbl_fix = 0;
12792                         if (dev_flow->skip_scale &
12793                                 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT))
12794                                 grp_info.skip_scale = 1;
12795                         else
12796                                 grp_info.skip_scale = 0;
12797                         ret = mlx5_flow_group_to_table(dev, tunnel,
12798                                                        jump_group,
12799                                                        &table,
12800                                                        &grp_info, error);
12801                         if (ret)
12802                                 return ret;
12803                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
12804                                                        attr->transfer,
12805                                                        !!dev_flow->external,
12806                                                        tunnel, jump_group, 0,
12807                                                        0, error);
12808                         if (!tbl)
12809                                 return rte_flow_error_set
12810                                                 (error, errno,
12811                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12812                                                  NULL,
12813                                                  "cannot create jump action.");
12814                         if (flow_dv_jump_tbl_resource_register
12815                             (dev, tbl, dev_flow, error)) {
12816                                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
12817                                 return rte_flow_error_set
12818                                                 (error, errno,
12819                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12820                                                  NULL,
12821                                                  "cannot create jump action.");
12822                         }
12823                         dev_flow->dv.actions[actions_n++] =
12824                                         dev_flow->dv.jump->action;
12825                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12826                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
12827                         sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP;
12828                         num_of_dest++;
12829                         break;
12830                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
12831                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
12832                         if (flow_dv_convert_action_modify_mac
12833                                         (mhdr_res, actions, error))
12834                                 return -rte_errno;
12835                         action_flags |= actions->type ==
12836                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
12837                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
12838                                         MLX5_FLOW_ACTION_SET_MAC_DST;
12839                         break;
12840                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
12841                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
12842                         if (flow_dv_convert_action_modify_ipv4
12843                                         (mhdr_res, actions, error))
12844                                 return -rte_errno;
12845                         action_flags |= actions->type ==
12846                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
12847                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
12848                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
12849                         break;
12850                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
12851                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
12852                         if (flow_dv_convert_action_modify_ipv6
12853                                         (mhdr_res, actions, error))
12854                                 return -rte_errno;
12855                         action_flags |= actions->type ==
12856                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
12857                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
12858                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
12859                         break;
12860                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
12861                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
12862                         if (flow_dv_convert_action_modify_tp
12863                                         (mhdr_res, actions, items,
12864                                          &flow_attr, dev_flow, !!(action_flags &
12865                                          MLX5_FLOW_ACTION_DECAP), error))
12866                                 return -rte_errno;
12867                         action_flags |= actions->type ==
12868                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
12869                                         MLX5_FLOW_ACTION_SET_TP_SRC :
12870                                         MLX5_FLOW_ACTION_SET_TP_DST;
12871                         break;
12872                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
12873                         if (flow_dv_convert_action_modify_dec_ttl
12874                                         (mhdr_res, items, &flow_attr, dev_flow,
12875                                          !!(action_flags &
12876                                          MLX5_FLOW_ACTION_DECAP), error))
12877                                 return -rte_errno;
12878                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
12879                         break;
12880                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
12881                         if (flow_dv_convert_action_modify_ttl
12882                                         (mhdr_res, actions, items, &flow_attr,
12883                                          dev_flow, !!(action_flags &
12884                                          MLX5_FLOW_ACTION_DECAP), error))
12885                                 return -rte_errno;
12886                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
12887                         break;
12888                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
12889                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
12890                         if (flow_dv_convert_action_modify_tcp_seq
12891                                         (mhdr_res, actions, error))
12892                                 return -rte_errno;
12893                         action_flags |= actions->type ==
12894                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
12895                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
12896                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
12897                         break;
12898
12899                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
12900                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
12901                         if (flow_dv_convert_action_modify_tcp_ack
12902                                         (mhdr_res, actions, error))
12903                                 return -rte_errno;
12904                         action_flags |= actions->type ==
12905                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
12906                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
12907                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
12908                         break;
12909                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
12910                         if (flow_dv_convert_action_set_reg
12911                                         (mhdr_res, actions, error))
12912                                 return -rte_errno;
12913                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12914                         break;
12915                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
12916                         if (flow_dv_convert_action_copy_mreg
12917                                         (dev, mhdr_res, actions, error))
12918                                 return -rte_errno;
12919                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12920                         break;
12921                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
12922                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
12923                         dev_flow->handle->fate_action =
12924                                         MLX5_FLOW_FATE_DEFAULT_MISS;
12925                         break;
12926                 case RTE_FLOW_ACTION_TYPE_METER:
12927                         if (!wks->fm)
12928                                 return rte_flow_error_set(error, rte_errno,
12929                                         RTE_FLOW_ERROR_TYPE_ACTION,
12930                                         NULL, "Failed to get meter in flow.");
12931                         /* Set the meter action. */
12932                         dev_flow->dv.actions[actions_n++] =
12933                                 wks->fm->meter_action;
12934                         action_flags |= MLX5_FLOW_ACTION_METER;
12935                         break;
12936                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
12937                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
12938                                                               actions, error))
12939                                 return -rte_errno;
12940                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
12941                         break;
12942                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
12943                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
12944                                                               actions, error))
12945                                 return -rte_errno;
12946                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
12947                         break;
12948                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
12949                         sample_act_pos = actions_n;
12950                         sample = (const struct rte_flow_action_sample *)
12951                                  action->conf;
12952                         actions_n++;
12953                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
12954                         /* put encap action into group if work with port id */
12955                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
12956                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
12957                                 sample_act->action_flags |=
12958                                                         MLX5_FLOW_ACTION_ENCAP;
12959                         break;
12960                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
12961                         if (flow_dv_convert_action_modify_field
12962                                         (dev, mhdr_res, actions, attr, error))
12963                                 return -rte_errno;
12964                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
12965                         break;
12966                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
12967                         owner_idx = (uint32_t)(uintptr_t)action->conf;
12968                         ct = flow_aso_ct_get_by_idx(dev, owner_idx);
12969                         if (!ct)
12970                                 return rte_flow_error_set(error, EINVAL,
12971                                                 RTE_FLOW_ERROR_TYPE_ACTION,
12972                                                 NULL,
12973                                                 "Failed to get CT object.");
12974                         if (mlx5_aso_ct_available(priv->sh, ct))
12975                                 return rte_flow_error_set(error, rte_errno,
12976                                                 RTE_FLOW_ERROR_TYPE_ACTION,
12977                                                 NULL,
12978                                                 "CT is unavailable.");
12979                         if (ct->is_original)
12980                                 dev_flow->dv.actions[actions_n] =
12981                                                         ct->dr_action_orig;
12982                         else
12983                                 dev_flow->dv.actions[actions_n] =
12984                                                         ct->dr_action_rply;
12985                         flow->indirect_type = MLX5_INDIRECT_ACTION_TYPE_CT;
12986                         flow->ct = owner_idx;
12987                         __atomic_fetch_add(&ct->refcnt, 1, __ATOMIC_RELAXED);
12988                         actions_n++;
12989                         action_flags |= MLX5_FLOW_ACTION_CT;
12990                         break;
12991                 case RTE_FLOW_ACTION_TYPE_END:
12992                         actions_end = true;
12993                         if (mhdr_res->actions_num) {
12994                                 /* create modify action if needed. */
12995                                 if (flow_dv_modify_hdr_resource_register
12996                                         (dev, mhdr_res, dev_flow, error))
12997                                         return -rte_errno;
12998                                 dev_flow->dv.actions[modify_action_position] =
12999                                         handle->dvh.modify_hdr->action;
13000                         }
13001                         /*
13002                          * Handle AGE and COUNT action by single HW counter
13003                          * when they are not shared.
13004                          */
13005                         if (action_flags & MLX5_FLOW_ACTION_AGE) {
13006                                 if ((non_shared_age &&
13007                                      count && !count->shared) ||
13008                                     !(priv->sh->flow_hit_aso_en &&
13009                                       (attr->group || attr->transfer))) {
13010                                         /* Creates age by counters. */
13011                                         cnt_act = flow_dv_prepare_counter
13012                                                                 (dev, dev_flow,
13013                                                                  flow, count,
13014                                                                  non_shared_age,
13015                                                                  error);
13016                                         if (!cnt_act)
13017                                                 return -rte_errno;
13018                                         dev_flow->dv.actions[age_act_pos] =
13019                                                                 cnt_act->action;
13020                                         break;
13021                                 }
13022                                 if (!flow->age && non_shared_age) {
13023                                         flow->age = flow_dv_aso_age_alloc
13024                                                                 (dev, error);
13025                                         if (!flow->age)
13026                                                 return -rte_errno;
13027                                         flow_dv_aso_age_params_init
13028                                                     (dev, flow->age,
13029                                                      non_shared_age->context ?
13030                                                      non_shared_age->context :
13031                                                      (void *)(uintptr_t)
13032                                                      (dev_flow->flow_idx),
13033                                                      non_shared_age->timeout);
13034                                 }
13035                                 age_act = flow_aso_age_get_by_idx(dev,
13036                                                                   flow->age);
13037                                 dev_flow->dv.actions[age_act_pos] =
13038                                                              age_act->dr_action;
13039                         }
13040                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
13041                                 /*
13042                                  * Create one count action, to be used
13043                                  * by all sub-flows.
13044                                  */
13045                                 cnt_act = flow_dv_prepare_counter(dev, dev_flow,
13046                                                                   flow, count,
13047                                                                   NULL, error);
13048                                 if (!cnt_act)
13049                                         return -rte_errno;
13050                                 dev_flow->dv.actions[actions_n++] =
13051                                                                 cnt_act->action;
13052                         }
13053                 default:
13054                         break;
13055                 }
13056                 if (mhdr_res->actions_num &&
13057                     modify_action_position == UINT32_MAX)
13058                         modify_action_position = actions_n++;
13059         }
13060         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
13061                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
13062                 int item_type = items->type;
13063
13064                 if (!mlx5_flow_os_item_supported(item_type))
13065                         return rte_flow_error_set(error, ENOTSUP,
13066                                                   RTE_FLOW_ERROR_TYPE_ITEM,
13067                                                   NULL, "item not supported");
13068                 switch (item_type) {
13069                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
13070                         flow_dv_translate_item_port_id
13071                                 (dev, match_mask, match_value, items, attr);
13072                         last_item = MLX5_FLOW_ITEM_PORT_ID;
13073                         break;
13074                 case RTE_FLOW_ITEM_TYPE_ETH:
13075                         flow_dv_translate_item_eth(match_mask, match_value,
13076                                                    items, tunnel,
13077                                                    dev_flow->dv.group);
13078                         matcher.priority = action_flags &
13079                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
13080                                         !dev_flow->external ?
13081                                         MLX5_PRIORITY_MAP_L3 :
13082                                         MLX5_PRIORITY_MAP_L2;
13083                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
13084                                              MLX5_FLOW_LAYER_OUTER_L2;
13085                         break;
13086                 case RTE_FLOW_ITEM_TYPE_VLAN:
13087                         flow_dv_translate_item_vlan(dev_flow,
13088                                                     match_mask, match_value,
13089                                                     items, tunnel,
13090                                                     dev_flow->dv.group);
13091                         matcher.priority = MLX5_PRIORITY_MAP_L2;
13092                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
13093                                               MLX5_FLOW_LAYER_INNER_VLAN) :
13094                                              (MLX5_FLOW_LAYER_OUTER_L2 |
13095                                               MLX5_FLOW_LAYER_OUTER_VLAN);
13096                         break;
13097                 case RTE_FLOW_ITEM_TYPE_IPV4:
13098                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13099                                                   &item_flags, &tunnel);
13100                         flow_dv_translate_item_ipv4(match_mask, match_value,
13101                                                     items, tunnel,
13102                                                     dev_flow->dv.group);
13103                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13104                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
13105                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
13106                         if (items->mask != NULL &&
13107                             ((const struct rte_flow_item_ipv4 *)
13108                              items->mask)->hdr.next_proto_id) {
13109                                 next_protocol =
13110                                         ((const struct rte_flow_item_ipv4 *)
13111                                          (items->spec))->hdr.next_proto_id;
13112                                 next_protocol &=
13113                                         ((const struct rte_flow_item_ipv4 *)
13114                                          (items->mask))->hdr.next_proto_id;
13115                         } else {
13116                                 /* Reset for inner layer. */
13117                                 next_protocol = 0xff;
13118                         }
13119                         break;
13120                 case RTE_FLOW_ITEM_TYPE_IPV6:
13121                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13122                                                   &item_flags, &tunnel);
13123                         flow_dv_translate_item_ipv6(match_mask, match_value,
13124                                                     items, tunnel,
13125                                                     dev_flow->dv.group);
13126                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13127                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
13128                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
13129                         if (items->mask != NULL &&
13130                             ((const struct rte_flow_item_ipv6 *)
13131                              items->mask)->hdr.proto) {
13132                                 next_protocol =
13133                                         ((const struct rte_flow_item_ipv6 *)
13134                                          items->spec)->hdr.proto;
13135                                 next_protocol &=
13136                                         ((const struct rte_flow_item_ipv6 *)
13137                                          items->mask)->hdr.proto;
13138                         } else {
13139                                 /* Reset for inner layer. */
13140                                 next_protocol = 0xff;
13141                         }
13142                         break;
13143                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
13144                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
13145                                                              match_value,
13146                                                              items, tunnel);
13147                         last_item = tunnel ?
13148                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
13149                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
13150                         if (items->mask != NULL &&
13151                             ((const struct rte_flow_item_ipv6_frag_ext *)
13152                              items->mask)->hdr.next_header) {
13153                                 next_protocol =
13154                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13155                                  items->spec)->hdr.next_header;
13156                                 next_protocol &=
13157                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13158                                  items->mask)->hdr.next_header;
13159                         } else {
13160                                 /* Reset for inner layer. */
13161                                 next_protocol = 0xff;
13162                         }
13163                         break;
13164                 case RTE_FLOW_ITEM_TYPE_TCP:
13165                         flow_dv_translate_item_tcp(match_mask, match_value,
13166                                                    items, tunnel);
13167                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13168                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
13169                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
13170                         break;
13171                 case RTE_FLOW_ITEM_TYPE_UDP:
13172                         flow_dv_translate_item_udp(match_mask, match_value,
13173                                                    items, tunnel);
13174                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13175                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
13176                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
13177                         break;
13178                 case RTE_FLOW_ITEM_TYPE_GRE:
13179                         flow_dv_translate_item_gre(match_mask, match_value,
13180                                                    items, tunnel);
13181                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13182                         last_item = MLX5_FLOW_LAYER_GRE;
13183                         break;
13184                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
13185                         flow_dv_translate_item_gre_key(match_mask,
13186                                                        match_value, items);
13187                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
13188                         break;
13189                 case RTE_FLOW_ITEM_TYPE_NVGRE:
13190                         flow_dv_translate_item_nvgre(match_mask, match_value,
13191                                                      items, tunnel);
13192                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13193                         last_item = MLX5_FLOW_LAYER_GRE;
13194                         break;
13195                 case RTE_FLOW_ITEM_TYPE_VXLAN:
13196                         flow_dv_translate_item_vxlan(dev, attr,
13197                                                      match_mask, match_value,
13198                                                      items, tunnel);
13199                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13200                         last_item = MLX5_FLOW_LAYER_VXLAN;
13201                         break;
13202                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
13203                         flow_dv_translate_item_vxlan_gpe(match_mask,
13204                                                          match_value, items,
13205                                                          tunnel);
13206                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13207                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
13208                         break;
13209                 case RTE_FLOW_ITEM_TYPE_GENEVE:
13210                         flow_dv_translate_item_geneve(match_mask, match_value,
13211                                                       items, tunnel);
13212                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13213                         last_item = MLX5_FLOW_LAYER_GENEVE;
13214                         break;
13215                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
13216                         ret = flow_dv_translate_item_geneve_opt(dev, match_mask,
13217                                                           match_value,
13218                                                           items, error);
13219                         if (ret)
13220                                 return rte_flow_error_set(error, -ret,
13221                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13222                                         "cannot create GENEVE TLV option");
13223                         flow->geneve_tlv_option = 1;
13224                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
13225                         break;
13226                 case RTE_FLOW_ITEM_TYPE_MPLS:
13227                         flow_dv_translate_item_mpls(match_mask, match_value,
13228                                                     items, last_item, tunnel);
13229                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13230                         last_item = MLX5_FLOW_LAYER_MPLS;
13231                         break;
13232                 case RTE_FLOW_ITEM_TYPE_MARK:
13233                         flow_dv_translate_item_mark(dev, match_mask,
13234                                                     match_value, items);
13235                         last_item = MLX5_FLOW_ITEM_MARK;
13236                         break;
13237                 case RTE_FLOW_ITEM_TYPE_META:
13238                         flow_dv_translate_item_meta(dev, match_mask,
13239                                                     match_value, attr, items);
13240                         last_item = MLX5_FLOW_ITEM_METADATA;
13241                         break;
13242                 case RTE_FLOW_ITEM_TYPE_ICMP:
13243                         flow_dv_translate_item_icmp(match_mask, match_value,
13244                                                     items, tunnel);
13245                         last_item = MLX5_FLOW_LAYER_ICMP;
13246                         break;
13247                 case RTE_FLOW_ITEM_TYPE_ICMP6:
13248                         flow_dv_translate_item_icmp6(match_mask, match_value,
13249                                                       items, tunnel);
13250                         last_item = MLX5_FLOW_LAYER_ICMP6;
13251                         break;
13252                 case RTE_FLOW_ITEM_TYPE_TAG:
13253                         flow_dv_translate_item_tag(dev, match_mask,
13254                                                    match_value, items);
13255                         last_item = MLX5_FLOW_ITEM_TAG;
13256                         break;
13257                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
13258                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
13259                                                         match_value, items);
13260                         last_item = MLX5_FLOW_ITEM_TAG;
13261                         break;
13262                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
13263                         flow_dv_translate_item_tx_queue(dev, match_mask,
13264                                                         match_value,
13265                                                         items);
13266                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
13267                         break;
13268                 case RTE_FLOW_ITEM_TYPE_GTP:
13269                         flow_dv_translate_item_gtp(match_mask, match_value,
13270                                                    items, tunnel);
13271                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13272                         last_item = MLX5_FLOW_LAYER_GTP;
13273                         break;
13274                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
13275                         ret = flow_dv_translate_item_gtp_psc(match_mask,
13276                                                           match_value,
13277                                                           items);
13278                         if (ret)
13279                                 return rte_flow_error_set(error, -ret,
13280                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13281                                         "cannot create GTP PSC item");
13282                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
13283                         break;
13284                 case RTE_FLOW_ITEM_TYPE_ECPRI:
13285                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
13286                                 /* Create it only the first time to be used. */
13287                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
13288                                 if (ret)
13289                                         return rte_flow_error_set
13290                                                 (error, -ret,
13291                                                 RTE_FLOW_ERROR_TYPE_ITEM,
13292                                                 NULL,
13293                                                 "cannot create eCPRI parser");
13294                         }
13295                         flow_dv_translate_item_ecpri(dev, match_mask,
13296                                                      match_value, items);
13297                         /* No other protocol should follow eCPRI layer. */
13298                         last_item = MLX5_FLOW_LAYER_ECPRI;
13299                         break;
13300                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
13301                         flow_dv_translate_item_integrity(match_mask,
13302                                                          match_value,
13303                                                          head_item, items);
13304                         break;
13305                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
13306                         flow_dv_translate_item_aso_ct(dev, match_mask,
13307                                                       match_value, items);
13308                         break;
13309                 default:
13310                         break;
13311                 }
13312                 item_flags |= last_item;
13313         }
13314         /*
13315          * When E-Switch mode is enabled, we have two cases where we need to
13316          * set the source port manually.
13317          * The first one, is in case of Nic steering rule, and the second is
13318          * E-Switch rule where no port_id item was found. In both cases
13319          * the source port is set according the current port in use.
13320          */
13321         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
13322             (priv->representor || priv->master)) {
13323                 if (flow_dv_translate_item_port_id(dev, match_mask,
13324                                                    match_value, NULL, attr))
13325                         return -rte_errno;
13326         }
13327 #ifdef RTE_LIBRTE_MLX5_DEBUG
13328         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
13329                                               dev_flow->dv.value.buf));
13330 #endif
13331         /*
13332          * Layers may be already initialized from prefix flow if this dev_flow
13333          * is the suffix flow.
13334          */
13335         handle->layers |= item_flags;
13336         if (action_flags & MLX5_FLOW_ACTION_RSS)
13337                 flow_dv_hashfields_set(dev_flow, rss_desc);
13338         /* If has RSS action in the sample action, the Sample/Mirror resource
13339          * should be registered after the hash filed be update.
13340          */
13341         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
13342                 ret = flow_dv_translate_action_sample(dev,
13343                                                       sample,
13344                                                       dev_flow, attr,
13345                                                       &num_of_dest,
13346                                                       sample_actions,
13347                                                       &sample_res,
13348                                                       error);
13349                 if (ret < 0)
13350                         return ret;
13351                 ret = flow_dv_create_action_sample(dev,
13352                                                    dev_flow,
13353                                                    num_of_dest,
13354                                                    &sample_res,
13355                                                    &mdest_res,
13356                                                    sample_actions,
13357                                                    action_flags,
13358                                                    error);
13359                 if (ret < 0)
13360                         return rte_flow_error_set
13361                                                 (error, rte_errno,
13362                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13363                                                 NULL,
13364                                                 "cannot create sample action");
13365                 if (num_of_dest > 1) {
13366                         dev_flow->dv.actions[sample_act_pos] =
13367                         dev_flow->dv.dest_array_res->action;
13368                 } else {
13369                         dev_flow->dv.actions[sample_act_pos] =
13370                         dev_flow->dv.sample_res->verbs_action;
13371                 }
13372         }
13373         /*
13374          * For multiple destination (sample action with ratio=1), the encap
13375          * action and port id action will be combined into group action.
13376          * So need remove the original these actions in the flow and only
13377          * use the sample action instead of.
13378          */
13379         if (num_of_dest > 1 &&
13380             (sample_act->dr_port_id_action || sample_act->dr_jump_action)) {
13381                 int i;
13382                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
13383
13384                 for (i = 0; i < actions_n; i++) {
13385                         if ((sample_act->dr_encap_action &&
13386                                 sample_act->dr_encap_action ==
13387                                 dev_flow->dv.actions[i]) ||
13388                                 (sample_act->dr_port_id_action &&
13389                                 sample_act->dr_port_id_action ==
13390                                 dev_flow->dv.actions[i]) ||
13391                                 (sample_act->dr_jump_action &&
13392                                 sample_act->dr_jump_action ==
13393                                 dev_flow->dv.actions[i]))
13394                                 continue;
13395                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
13396                 }
13397                 memcpy((void *)dev_flow->dv.actions,
13398                                 (void *)temp_actions,
13399                                 tmp_actions_n * sizeof(void *));
13400                 actions_n = tmp_actions_n;
13401         }
13402         dev_flow->dv.actions_n = actions_n;
13403         dev_flow->act_flags = action_flags;
13404         if (wks->skip_matcher_reg)
13405                 return 0;
13406         /* Register matcher. */
13407         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
13408                                     matcher.mask.size);
13409         matcher.priority = mlx5_get_matcher_priority(dev, attr,
13410                                         matcher.priority);
13411         /**
13412          * When creating meter drop flow in drop table, using original
13413          * 5-tuple match, the matcher priority should be lower than
13414          * mtr_id matcher.
13415          */
13416         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
13417             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP &&
13418             matcher.priority <= MLX5_REG_BITS)
13419                 matcher.priority += MLX5_REG_BITS;
13420         /* reserved field no needs to be set to 0 here. */
13421         tbl_key.is_fdb = attr->transfer;
13422         tbl_key.is_egress = attr->egress;
13423         tbl_key.level = dev_flow->dv.group;
13424         tbl_key.id = dev_flow->dv.table_id;
13425         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
13426                                      tunnel, attr->group, error))
13427                 return -rte_errno;
13428         return 0;
13429 }
13430
13431 /**
13432  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13433  * and tunnel.
13434  *
13435  * @param[in, out] action
13436  *   Shred RSS action holding hash RX queue objects.
13437  * @param[in] hash_fields
13438  *   Defines combination of packet fields to participate in RX hash.
13439  * @param[in] tunnel
13440  *   Tunnel type
13441  * @param[in] hrxq_idx
13442  *   Hash RX queue index to set.
13443  *
13444  * @return
13445  *   0 on success, otherwise negative errno value.
13446  */
13447 static int
13448 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
13449                               const uint64_t hash_fields,
13450                               uint32_t hrxq_idx)
13451 {
13452         uint32_t *hrxqs = action->hrxq;
13453
13454         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13455         case MLX5_RSS_HASH_IPV4:
13456                 /* fall-through. */
13457         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13458                 /* fall-through. */
13459         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13460                 hrxqs[0] = hrxq_idx;
13461                 return 0;
13462         case MLX5_RSS_HASH_IPV4_TCP:
13463                 /* fall-through. */
13464         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13465                 /* fall-through. */
13466         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13467                 hrxqs[1] = hrxq_idx;
13468                 return 0;
13469         case MLX5_RSS_HASH_IPV4_UDP:
13470                 /* fall-through. */
13471         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13472                 /* fall-through. */
13473         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13474                 hrxqs[2] = hrxq_idx;
13475                 return 0;
13476         case MLX5_RSS_HASH_IPV6:
13477                 /* fall-through. */
13478         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13479                 /* fall-through. */
13480         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13481                 hrxqs[3] = hrxq_idx;
13482                 return 0;
13483         case MLX5_RSS_HASH_IPV6_TCP:
13484                 /* fall-through. */
13485         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13486                 /* fall-through. */
13487         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13488                 hrxqs[4] = hrxq_idx;
13489                 return 0;
13490         case MLX5_RSS_HASH_IPV6_UDP:
13491                 /* fall-through. */
13492         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13493                 /* fall-through. */
13494         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13495                 hrxqs[5] = hrxq_idx;
13496                 return 0;
13497         case MLX5_RSS_HASH_NONE:
13498                 hrxqs[6] = hrxq_idx;
13499                 return 0;
13500         default:
13501                 return -1;
13502         }
13503 }
13504
13505 /**
13506  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13507  * and tunnel.
13508  *
13509  * @param[in] dev
13510  *   Pointer to the Ethernet device structure.
13511  * @param[in] idx
13512  *   Shared RSS action ID holding hash RX queue objects.
13513  * @param[in] hash_fields
13514  *   Defines combination of packet fields to participate in RX hash.
13515  * @param[in] tunnel
13516  *   Tunnel type
13517  *
13518  * @return
13519  *   Valid hash RX queue index, otherwise 0.
13520  */
13521 static uint32_t
13522 __flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
13523                                  const uint64_t hash_fields)
13524 {
13525         struct mlx5_priv *priv = dev->data->dev_private;
13526         struct mlx5_shared_action_rss *shared_rss =
13527             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
13528         const uint32_t *hrxqs = shared_rss->hrxq;
13529
13530         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13531         case MLX5_RSS_HASH_IPV4:
13532                 /* fall-through. */
13533         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13534                 /* fall-through. */
13535         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13536                 return hrxqs[0];
13537         case MLX5_RSS_HASH_IPV4_TCP:
13538                 /* fall-through. */
13539         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13540                 /* fall-through. */
13541         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13542                 return hrxqs[1];
13543         case MLX5_RSS_HASH_IPV4_UDP:
13544                 /* fall-through. */
13545         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13546                 /* fall-through. */
13547         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13548                 return hrxqs[2];
13549         case MLX5_RSS_HASH_IPV6:
13550                 /* fall-through. */
13551         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13552                 /* fall-through. */
13553         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13554                 return hrxqs[3];
13555         case MLX5_RSS_HASH_IPV6_TCP:
13556                 /* fall-through. */
13557         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13558                 /* fall-through. */
13559         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13560                 return hrxqs[4];
13561         case MLX5_RSS_HASH_IPV6_UDP:
13562                 /* fall-through. */
13563         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13564                 /* fall-through. */
13565         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13566                 return hrxqs[5];
13567         case MLX5_RSS_HASH_NONE:
13568                 return hrxqs[6];
13569         default:
13570                 return 0;
13571         }
13572
13573 }
13574
13575 /**
13576  * Apply the flow to the NIC, lock free,
13577  * (mutex should be acquired by caller).
13578  *
13579  * @param[in] dev
13580  *   Pointer to the Ethernet device structure.
13581  * @param[in, out] flow
13582  *   Pointer to flow structure.
13583  * @param[out] error
13584  *   Pointer to error structure.
13585  *
13586  * @return
13587  *   0 on success, a negative errno value otherwise and rte_errno is set.
13588  */
13589 static int
13590 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
13591               struct rte_flow_error *error)
13592 {
13593         struct mlx5_flow_dv_workspace *dv;
13594         struct mlx5_flow_handle *dh;
13595         struct mlx5_flow_handle_dv *dv_h;
13596         struct mlx5_flow *dev_flow;
13597         struct mlx5_priv *priv = dev->data->dev_private;
13598         uint32_t handle_idx;
13599         int n;
13600         int err;
13601         int idx;
13602         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
13603         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
13604         uint8_t misc_mask;
13605
13606         MLX5_ASSERT(wks);
13607         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
13608                 dev_flow = &wks->flows[idx];
13609                 dv = &dev_flow->dv;
13610                 dh = dev_flow->handle;
13611                 dv_h = &dh->dvh;
13612                 n = dv->actions_n;
13613                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
13614                         if (dv->transfer) {
13615                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13616                                 dv->actions[n++] = priv->sh->dr_drop_action;
13617                         } else {
13618 #ifdef HAVE_MLX5DV_DR
13619                                 /* DR supports drop action placeholder. */
13620                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13621                                 dv->actions[n++] = priv->sh->dr_drop_action;
13622 #else
13623                                 /* For DV we use the explicit drop queue. */
13624                                 MLX5_ASSERT(priv->drop_queue.hrxq);
13625                                 dv->actions[n++] =
13626                                                 priv->drop_queue.hrxq->action;
13627 #endif
13628                         }
13629                 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
13630                            !dv_h->rix_sample && !dv_h->rix_dest_array)) {
13631                         struct mlx5_hrxq *hrxq;
13632                         uint32_t hrxq_idx;
13633
13634                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc,
13635                                                     &hrxq_idx);
13636                         if (!hrxq) {
13637                                 rte_flow_error_set
13638                                         (error, rte_errno,
13639                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13640                                          "cannot get hash queue");
13641                                 goto error;
13642                         }
13643                         dh->rix_hrxq = hrxq_idx;
13644                         dv->actions[n++] = hrxq->action;
13645                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13646                         struct mlx5_hrxq *hrxq = NULL;
13647                         uint32_t hrxq_idx;
13648
13649                         hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev,
13650                                                 rss_desc->shared_rss,
13651                                                 dev_flow->hash_fields);
13652                         if (hrxq_idx)
13653                                 hrxq = mlx5_ipool_get
13654                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
13655                                          hrxq_idx);
13656                         if (!hrxq) {
13657                                 rte_flow_error_set
13658                                         (error, rte_errno,
13659                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13660                                          "cannot get hash queue");
13661                                 goto error;
13662                         }
13663                         dh->rix_srss = rss_desc->shared_rss;
13664                         dv->actions[n++] = hrxq->action;
13665                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
13666                         if (!priv->sh->default_miss_action) {
13667                                 rte_flow_error_set
13668                                         (error, rte_errno,
13669                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13670                                          "default miss action not be created.");
13671                                 goto error;
13672                         }
13673                         dv->actions[n++] = priv->sh->default_miss_action;
13674                 }
13675                 misc_mask = flow_dv_matcher_enable(dv->value.buf);
13676                 __flow_dv_adjust_buf_size(&dv->value.size, misc_mask);
13677                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
13678                                                (void *)&dv->value, n,
13679                                                dv->actions, &dh->drv_flow);
13680                 if (err) {
13681                         rte_flow_error_set
13682                                 (error, errno,
13683                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
13684                                 NULL,
13685                                 (!priv->config.allow_duplicate_pattern &&
13686                                 errno == EEXIST) ?
13687                                 "duplicating pattern is not allowed" :
13688                                 "hardware refuses to create flow");
13689                         goto error;
13690                 }
13691                 if (priv->vmwa_context &&
13692                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
13693                         /*
13694                          * The rule contains the VLAN pattern.
13695                          * For VF we are going to create VLAN
13696                          * interface to make hypervisor set correct
13697                          * e-Switch vport context.
13698                          */
13699                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
13700                 }
13701         }
13702         return 0;
13703 error:
13704         err = rte_errno; /* Save rte_errno before cleanup. */
13705         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
13706                        handle_idx, dh, next) {
13707                 /* hrxq is union, don't clear it if the flag is not set. */
13708                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
13709                         mlx5_hrxq_release(dev, dh->rix_hrxq);
13710                         dh->rix_hrxq = 0;
13711                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13712                         dh->rix_srss = 0;
13713                 }
13714                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
13715                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
13716         }
13717         rte_errno = err; /* Restore rte_errno. */
13718         return -rte_errno;
13719 }
13720
13721 void
13722 flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
13723                           struct mlx5_list_entry *entry)
13724 {
13725         struct mlx5_flow_dv_matcher *resource = container_of(entry,
13726                                                              typeof(*resource),
13727                                                              entry);
13728
13729         claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
13730         mlx5_free(resource);
13731 }
13732
13733 /**
13734  * Release the flow matcher.
13735  *
13736  * @param dev
13737  *   Pointer to Ethernet device.
13738  * @param port_id
13739  *   Index to port ID action resource.
13740  *
13741  * @return
13742  *   1 while a reference on it exists, 0 when freed.
13743  */
13744 static int
13745 flow_dv_matcher_release(struct rte_eth_dev *dev,
13746                         struct mlx5_flow_handle *handle)
13747 {
13748         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
13749         struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl,
13750                                                             typeof(*tbl), tbl);
13751         int ret;
13752
13753         MLX5_ASSERT(matcher->matcher_object);
13754         ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
13755         flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
13756         return ret;
13757 }
13758
13759 void
13760 flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13761 {
13762         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13763         struct mlx5_flow_dv_encap_decap_resource *res =
13764                                        container_of(entry, typeof(*res), entry);
13765
13766         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13767         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
13768 }
13769
13770 /**
13771  * Release an encap/decap resource.
13772  *
13773  * @param dev
13774  *   Pointer to Ethernet device.
13775  * @param encap_decap_idx
13776  *   Index of encap decap resource.
13777  *
13778  * @return
13779  *   1 while a reference on it exists, 0 when freed.
13780  */
13781 static int
13782 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
13783                                      uint32_t encap_decap_idx)
13784 {
13785         struct mlx5_priv *priv = dev->data->dev_private;
13786         struct mlx5_flow_dv_encap_decap_resource *resource;
13787
13788         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
13789                                   encap_decap_idx);
13790         if (!resource)
13791                 return 0;
13792         MLX5_ASSERT(resource->action);
13793         return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
13794 }
13795
13796 /**
13797  * Release an jump to table action resource.
13798  *
13799  * @param dev
13800  *   Pointer to Ethernet device.
13801  * @param rix_jump
13802  *   Index to the jump action resource.
13803  *
13804  * @return
13805  *   1 while a reference on it exists, 0 when freed.
13806  */
13807 static int
13808 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
13809                                   uint32_t rix_jump)
13810 {
13811         struct mlx5_priv *priv = dev->data->dev_private;
13812         struct mlx5_flow_tbl_data_entry *tbl_data;
13813
13814         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
13815                                   rix_jump);
13816         if (!tbl_data)
13817                 return 0;
13818         return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl);
13819 }
13820
13821 void
13822 flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
13823                          struct mlx5_list_entry *entry)
13824 {
13825         struct mlx5_flow_dv_modify_hdr_resource *res =
13826                 container_of(entry, typeof(*res), entry);
13827
13828         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13829         mlx5_free(entry);
13830 }
13831
13832 /**
13833  * Release a modify-header resource.
13834  *
13835  * @param dev
13836  *   Pointer to Ethernet device.
13837  * @param handle
13838  *   Pointer to mlx5_flow_handle.
13839  *
13840  * @return
13841  *   1 while a reference on it exists, 0 when freed.
13842  */
13843 static int
13844 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
13845                                     struct mlx5_flow_handle *handle)
13846 {
13847         struct mlx5_priv *priv = dev->data->dev_private;
13848         struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr;
13849
13850         MLX5_ASSERT(entry->action);
13851         return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry);
13852 }
13853
13854 void
13855 flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13856 {
13857         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13858         struct mlx5_flow_dv_port_id_action_resource *resource =
13859                                   container_of(entry, typeof(*resource), entry);
13860
13861         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
13862         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
13863 }
13864
13865 /**
13866  * Release port ID action resource.
13867  *
13868  * @param dev
13869  *   Pointer to Ethernet device.
13870  * @param handle
13871  *   Pointer to mlx5_flow_handle.
13872  *
13873  * @return
13874  *   1 while a reference on it exists, 0 when freed.
13875  */
13876 static int
13877 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
13878                                         uint32_t port_id)
13879 {
13880         struct mlx5_priv *priv = dev->data->dev_private;
13881         struct mlx5_flow_dv_port_id_action_resource *resource;
13882
13883         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
13884         if (!resource)
13885                 return 0;
13886         MLX5_ASSERT(resource->action);
13887         return mlx5_list_unregister(priv->sh->port_id_action_list,
13888                                     &resource->entry);
13889 }
13890
13891 /**
13892  * Release shared RSS action resource.
13893  *
13894  * @param dev
13895  *   Pointer to Ethernet device.
13896  * @param srss
13897  *   Shared RSS action index.
13898  */
13899 static void
13900 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
13901 {
13902         struct mlx5_priv *priv = dev->data->dev_private;
13903         struct mlx5_shared_action_rss *shared_rss;
13904
13905         shared_rss = mlx5_ipool_get
13906                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss);
13907         __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
13908 }
13909
13910 void
13911 flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13912 {
13913         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13914         struct mlx5_flow_dv_push_vlan_action_resource *resource =
13915                         container_of(entry, typeof(*resource), entry);
13916
13917         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
13918         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
13919 }
13920
13921 /**
13922  * Release push vlan action resource.
13923  *
13924  * @param dev
13925  *   Pointer to Ethernet device.
13926  * @param handle
13927  *   Pointer to mlx5_flow_handle.
13928  *
13929  * @return
13930  *   1 while a reference on it exists, 0 when freed.
13931  */
13932 static int
13933 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
13934                                           struct mlx5_flow_handle *handle)
13935 {
13936         struct mlx5_priv *priv = dev->data->dev_private;
13937         struct mlx5_flow_dv_push_vlan_action_resource *resource;
13938         uint32_t idx = handle->dvh.rix_push_vlan;
13939
13940         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
13941         if (!resource)
13942                 return 0;
13943         MLX5_ASSERT(resource->action);
13944         return mlx5_list_unregister(priv->sh->push_vlan_action_list,
13945                                     &resource->entry);
13946 }
13947
13948 /**
13949  * Release the fate resource.
13950  *
13951  * @param dev
13952  *   Pointer to Ethernet device.
13953  * @param handle
13954  *   Pointer to mlx5_flow_handle.
13955  */
13956 static void
13957 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
13958                                struct mlx5_flow_handle *handle)
13959 {
13960         if (!handle->rix_fate)
13961                 return;
13962         switch (handle->fate_action) {
13963         case MLX5_FLOW_FATE_QUEUE:
13964                 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array)
13965                         mlx5_hrxq_release(dev, handle->rix_hrxq);
13966                 break;
13967         case MLX5_FLOW_FATE_JUMP:
13968                 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump);
13969                 break;
13970         case MLX5_FLOW_FATE_PORT_ID:
13971                 flow_dv_port_id_action_resource_release(dev,
13972                                 handle->rix_port_id_action);
13973                 break;
13974         default:
13975                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
13976                 break;
13977         }
13978         handle->rix_fate = 0;
13979 }
13980
13981 void
13982 flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
13983                          struct mlx5_list_entry *entry)
13984 {
13985         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
13986                                                               typeof(*resource),
13987                                                               entry);
13988         struct rte_eth_dev *dev = resource->dev;
13989         struct mlx5_priv *priv = dev->data->dev_private;
13990
13991         if (resource->verbs_action)
13992                 claim_zero(mlx5_flow_os_destroy_flow_action
13993                                                       (resource->verbs_action));
13994         if (resource->normal_path_tbl)
13995                 flow_dv_tbl_resource_release(MLX5_SH(dev),
13996                                              resource->normal_path_tbl);
13997         flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
13998         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
13999         DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
14000 }
14001
14002 /**
14003  * Release an sample resource.
14004  *
14005  * @param dev
14006  *   Pointer to Ethernet device.
14007  * @param handle
14008  *   Pointer to mlx5_flow_handle.
14009  *
14010  * @return
14011  *   1 while a reference on it exists, 0 when freed.
14012  */
14013 static int
14014 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
14015                                      struct mlx5_flow_handle *handle)
14016 {
14017         struct mlx5_priv *priv = dev->data->dev_private;
14018         struct mlx5_flow_dv_sample_resource *resource;
14019
14020         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
14021                                   handle->dvh.rix_sample);
14022         if (!resource)
14023                 return 0;
14024         MLX5_ASSERT(resource->verbs_action);
14025         return mlx5_list_unregister(priv->sh->sample_action_list,
14026                                     &resource->entry);
14027 }
14028
14029 void
14030 flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
14031                              struct mlx5_list_entry *entry)
14032 {
14033         struct mlx5_flow_dv_dest_array_resource *resource =
14034                         container_of(entry, typeof(*resource), entry);
14035         struct rte_eth_dev *dev = resource->dev;
14036         struct mlx5_priv *priv = dev->data->dev_private;
14037         uint32_t i = 0;
14038
14039         MLX5_ASSERT(resource->action);
14040         if (resource->action)
14041                 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14042         for (; i < resource->num_of_dest; i++)
14043                 flow_dv_sample_sub_actions_release(dev,
14044                                                    &resource->sample_idx[i]);
14045         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
14046         DRV_LOG(DEBUG, "destination array resource %p: removed",
14047                 (void *)resource);
14048 }
14049
14050 /**
14051  * Release an destination array resource.
14052  *
14053  * @param dev
14054  *   Pointer to Ethernet device.
14055  * @param handle
14056  *   Pointer to mlx5_flow_handle.
14057  *
14058  * @return
14059  *   1 while a reference on it exists, 0 when freed.
14060  */
14061 static int
14062 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
14063                                     struct mlx5_flow_handle *handle)
14064 {
14065         struct mlx5_priv *priv = dev->data->dev_private;
14066         struct mlx5_flow_dv_dest_array_resource *resource;
14067
14068         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
14069                                   handle->dvh.rix_dest_array);
14070         if (!resource)
14071                 return 0;
14072         MLX5_ASSERT(resource->action);
14073         return mlx5_list_unregister(priv->sh->dest_array_list,
14074                                     &resource->entry);
14075 }
14076
14077 static void
14078 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev)
14079 {
14080         struct mlx5_priv *priv = dev->data->dev_private;
14081         struct mlx5_dev_ctx_shared *sh = priv->sh;
14082         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
14083                                 sh->geneve_tlv_option_resource;
14084         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
14085         if (geneve_opt_resource) {
14086                 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1,
14087                                          __ATOMIC_RELAXED))) {
14088                         claim_zero(mlx5_devx_cmd_destroy
14089                                         (geneve_opt_resource->obj));
14090                         mlx5_free(sh->geneve_tlv_option_resource);
14091                         sh->geneve_tlv_option_resource = NULL;
14092                 }
14093         }
14094         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
14095 }
14096
14097 /**
14098  * Remove the flow from the NIC but keeps it in memory.
14099  * Lock free, (mutex should be acquired by caller).
14100  *
14101  * @param[in] dev
14102  *   Pointer to Ethernet device.
14103  * @param[in, out] flow
14104  *   Pointer to flow structure.
14105  */
14106 static void
14107 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
14108 {
14109         struct mlx5_flow_handle *dh;
14110         uint32_t handle_idx;
14111         struct mlx5_priv *priv = dev->data->dev_private;
14112
14113         if (!flow)
14114                 return;
14115         handle_idx = flow->dev_handles;
14116         while (handle_idx) {
14117                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14118                                     handle_idx);
14119                 if (!dh)
14120                         return;
14121                 if (dh->drv_flow) {
14122                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
14123                         dh->drv_flow = NULL;
14124                 }
14125                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
14126                         flow_dv_fate_resource_release(dev, dh);
14127                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
14128                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
14129                 handle_idx = dh->next.next;
14130         }
14131 }
14132
14133 /**
14134  * Remove the flow from the NIC and the memory.
14135  * Lock free, (mutex should be acquired by caller).
14136  *
14137  * @param[in] dev
14138  *   Pointer to the Ethernet device structure.
14139  * @param[in, out] flow
14140  *   Pointer to flow structure.
14141  */
14142 static void
14143 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
14144 {
14145         struct mlx5_flow_handle *dev_handle;
14146         struct mlx5_priv *priv = dev->data->dev_private;
14147         struct mlx5_flow_meter_info *fm = NULL;
14148         uint32_t srss = 0;
14149
14150         if (!flow)
14151                 return;
14152         flow_dv_remove(dev, flow);
14153         if (flow->counter) {
14154                 flow_dv_counter_free(dev, flow->counter);
14155                 flow->counter = 0;
14156         }
14157         if (flow->meter) {
14158                 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
14159                 if (fm)
14160                         mlx5_flow_meter_detach(priv, fm);
14161                 flow->meter = 0;
14162         }
14163         /* Keep the current age handling by default. */
14164         if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct)
14165                 flow_dv_aso_ct_release(dev, flow->ct);
14166         else if (flow->age)
14167                 flow_dv_aso_age_release(dev, flow->age);
14168         if (flow->geneve_tlv_option) {
14169                 flow_dv_geneve_tlv_option_resource_release(dev);
14170                 flow->geneve_tlv_option = 0;
14171         }
14172         while (flow->dev_handles) {
14173                 uint32_t tmp_idx = flow->dev_handles;
14174
14175                 dev_handle = mlx5_ipool_get(priv->sh->ipool
14176                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
14177                 if (!dev_handle)
14178                         return;
14179                 flow->dev_handles = dev_handle->next.next;
14180                 if (dev_handle->dvh.matcher)
14181                         flow_dv_matcher_release(dev, dev_handle);
14182                 if (dev_handle->dvh.rix_sample)
14183                         flow_dv_sample_resource_release(dev, dev_handle);
14184                 if (dev_handle->dvh.rix_dest_array)
14185                         flow_dv_dest_array_resource_release(dev, dev_handle);
14186                 if (dev_handle->dvh.rix_encap_decap)
14187                         flow_dv_encap_decap_resource_release(dev,
14188                                 dev_handle->dvh.rix_encap_decap);
14189                 if (dev_handle->dvh.modify_hdr)
14190                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
14191                 if (dev_handle->dvh.rix_push_vlan)
14192                         flow_dv_push_vlan_action_resource_release(dev,
14193                                                                   dev_handle);
14194                 if (dev_handle->dvh.rix_tag)
14195                         flow_dv_tag_release(dev,
14196                                             dev_handle->dvh.rix_tag);
14197                 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS)
14198                         flow_dv_fate_resource_release(dev, dev_handle);
14199                 else if (!srss)
14200                         srss = dev_handle->rix_srss;
14201                 if (fm && dev_handle->is_meter_flow_id &&
14202                     dev_handle->split_flow_id)
14203                         mlx5_ipool_free(fm->flow_ipool,
14204                                         dev_handle->split_flow_id);
14205                 else if (dev_handle->split_flow_id &&
14206                     !dev_handle->is_meter_flow_id)
14207                         mlx5_ipool_free(priv->sh->ipool
14208                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
14209                                         dev_handle->split_flow_id);
14210                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14211                            tmp_idx);
14212         }
14213         if (srss)
14214                 flow_dv_shared_rss_action_release(dev, srss);
14215 }
14216
14217 /**
14218  * Release array of hash RX queue objects.
14219  * Helper function.
14220  *
14221  * @param[in] dev
14222  *   Pointer to the Ethernet device structure.
14223  * @param[in, out] hrxqs
14224  *   Array of hash RX queue objects.
14225  *
14226  * @return
14227  *   Total number of references to hash RX queue objects in *hrxqs* array
14228  *   after this operation.
14229  */
14230 static int
14231 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
14232                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
14233 {
14234         size_t i;
14235         int remaining = 0;
14236
14237         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
14238                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
14239
14240                 if (!ret)
14241                         (*hrxqs)[i] = 0;
14242                 remaining += ret;
14243         }
14244         return remaining;
14245 }
14246
14247 /**
14248  * Release all hash RX queue objects representing shared RSS action.
14249  *
14250  * @param[in] dev
14251  *   Pointer to the Ethernet device structure.
14252  * @param[in, out] action
14253  *   Shared RSS action to remove hash RX queue objects from.
14254  *
14255  * @return
14256  *   Total number of references to hash RX queue objects stored in *action*
14257  *   after this operation.
14258  *   Expected to be 0 if no external references held.
14259  */
14260 static int
14261 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
14262                                  struct mlx5_shared_action_rss *shared_rss)
14263 {
14264         return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq);
14265 }
14266
14267 /**
14268  * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to
14269  * user input.
14270  *
14271  * Only one hash value is available for one L3+L4 combination:
14272  * for example:
14273  * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and
14274  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
14275  * same slot in mlx5_rss_hash_fields.
14276  *
14277  * @param[in] rss
14278  *   Pointer to the shared action RSS conf.
14279  * @param[in, out] hash_field
14280  *   hash_field variable needed to be adjusted.
14281  *
14282  * @return
14283  *   void
14284  */
14285 static void
14286 __flow_dv_action_rss_l34_hash_adjust(struct mlx5_shared_action_rss *rss,
14287                                      uint64_t *hash_field)
14288 {
14289         uint64_t rss_types = rss->origin.types;
14290
14291         switch (*hash_field & ~IBV_RX_HASH_INNER) {
14292         case MLX5_RSS_HASH_IPV4:
14293                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
14294                         *hash_field &= ~MLX5_RSS_HASH_IPV4;
14295                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14296                                 *hash_field |= IBV_RX_HASH_DST_IPV4;
14297                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14298                                 *hash_field |= IBV_RX_HASH_SRC_IPV4;
14299                         else
14300                                 *hash_field |= MLX5_RSS_HASH_IPV4;
14301                 }
14302                 return;
14303         case MLX5_RSS_HASH_IPV6:
14304                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
14305                         *hash_field &= ~MLX5_RSS_HASH_IPV6;
14306                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14307                                 *hash_field |= IBV_RX_HASH_DST_IPV6;
14308                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14309                                 *hash_field |= IBV_RX_HASH_SRC_IPV6;
14310                         else
14311                                 *hash_field |= MLX5_RSS_HASH_IPV6;
14312                 }
14313                 return;
14314         case MLX5_RSS_HASH_IPV4_UDP:
14315                 /* fall-through. */
14316         case MLX5_RSS_HASH_IPV6_UDP:
14317                 if (rss_types & ETH_RSS_UDP) {
14318                         *hash_field &= ~MLX5_UDP_IBV_RX_HASH;
14319                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14320                                 *hash_field |= IBV_RX_HASH_DST_PORT_UDP;
14321                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14322                                 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP;
14323                         else
14324                                 *hash_field |= MLX5_UDP_IBV_RX_HASH;
14325                 }
14326                 return;
14327         case MLX5_RSS_HASH_IPV4_TCP:
14328                 /* fall-through. */
14329         case MLX5_RSS_HASH_IPV6_TCP:
14330                 if (rss_types & ETH_RSS_TCP) {
14331                         *hash_field &= ~MLX5_TCP_IBV_RX_HASH;
14332                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14333                                 *hash_field |= IBV_RX_HASH_DST_PORT_TCP;
14334                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14335                                 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP;
14336                         else
14337                                 *hash_field |= MLX5_TCP_IBV_RX_HASH;
14338                 }
14339                 return;
14340         default:
14341                 return;
14342         }
14343 }
14344
14345 /**
14346  * Setup shared RSS action.
14347  * Prepare set of hash RX queue objects sufficient to handle all valid
14348  * hash_fields combinations (see enum ibv_rx_hash_fields).
14349  *
14350  * @param[in] dev
14351  *   Pointer to the Ethernet device structure.
14352  * @param[in] action_idx
14353  *   Shared RSS action ipool index.
14354  * @param[in, out] action
14355  *   Partially initialized shared RSS action.
14356  * @param[out] error
14357  *   Perform verbose error reporting if not NULL. Initialized in case of
14358  *   error only.
14359  *
14360  * @return
14361  *   0 on success, otherwise negative errno value.
14362  */
14363 static int
14364 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
14365                            uint32_t action_idx,
14366                            struct mlx5_shared_action_rss *shared_rss,
14367                            struct rte_flow_error *error)
14368 {
14369         struct mlx5_flow_rss_desc rss_desc = { 0 };
14370         size_t i;
14371         int err;
14372
14373         if (mlx5_ind_table_obj_setup(dev, shared_rss->ind_tbl)) {
14374                 return rte_flow_error_set(error, rte_errno,
14375                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14376                                           "cannot setup indirection table");
14377         }
14378         memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN);
14379         rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
14380         rss_desc.const_q = shared_rss->origin.queue;
14381         rss_desc.queue_num = shared_rss->origin.queue_num;
14382         /* Set non-zero value to indicate a shared RSS. */
14383         rss_desc.shared_rss = action_idx;
14384         rss_desc.ind_tbl = shared_rss->ind_tbl;
14385         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
14386                 uint32_t hrxq_idx;
14387                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
14388                 int tunnel = 0;
14389
14390                 __flow_dv_action_rss_l34_hash_adjust(shared_rss, &hash_fields);
14391                 if (shared_rss->origin.level > 1) {
14392                         hash_fields |= IBV_RX_HASH_INNER;
14393                         tunnel = 1;
14394                 }
14395                 rss_desc.tunnel = tunnel;
14396                 rss_desc.hash_fields = hash_fields;
14397                 hrxq_idx = mlx5_hrxq_get(dev, &rss_desc);
14398                 if (!hrxq_idx) {
14399                         rte_flow_error_set
14400                                 (error, rte_errno,
14401                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14402                                  "cannot get hash queue");
14403                         goto error_hrxq_new;
14404                 }
14405                 err = __flow_dv_action_rss_hrxq_set
14406                         (shared_rss, hash_fields, hrxq_idx);
14407                 MLX5_ASSERT(!err);
14408         }
14409         return 0;
14410 error_hrxq_new:
14411         err = rte_errno;
14412         __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14413         if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true))
14414                 shared_rss->ind_tbl = NULL;
14415         rte_errno = err;
14416         return -rte_errno;
14417 }
14418
14419 /**
14420  * Create shared RSS action.
14421  *
14422  * @param[in] dev
14423  *   Pointer to the Ethernet device structure.
14424  * @param[in] conf
14425  *   Shared action configuration.
14426  * @param[in] rss
14427  *   RSS action specification used to create shared action.
14428  * @param[out] error
14429  *   Perform verbose error reporting if not NULL. Initialized in case of
14430  *   error only.
14431  *
14432  * @return
14433  *   A valid shared action ID in case of success, 0 otherwise and
14434  *   rte_errno is set.
14435  */
14436 static uint32_t
14437 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
14438                             const struct rte_flow_indir_action_conf *conf,
14439                             const struct rte_flow_action_rss *rss,
14440                             struct rte_flow_error *error)
14441 {
14442         struct mlx5_priv *priv = dev->data->dev_private;
14443         struct mlx5_shared_action_rss *shared_rss = NULL;
14444         void *queue = NULL;
14445         struct rte_flow_action_rss *origin;
14446         const uint8_t *rss_key;
14447         uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
14448         uint32_t idx;
14449
14450         RTE_SET_USED(conf);
14451         queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14452                             0, SOCKET_ID_ANY);
14453         shared_rss = mlx5_ipool_zmalloc
14454                          (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx);
14455         if (!shared_rss || !queue) {
14456                 rte_flow_error_set(error, ENOMEM,
14457                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14458                                    "cannot allocate resource memory");
14459                 goto error_rss_init;
14460         }
14461         if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) {
14462                 rte_flow_error_set(error, E2BIG,
14463                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14464                                    "rss action number out of range");
14465                 goto error_rss_init;
14466         }
14467         shared_rss->ind_tbl = mlx5_malloc(MLX5_MEM_ZERO,
14468                                           sizeof(*shared_rss->ind_tbl),
14469                                           0, SOCKET_ID_ANY);
14470         if (!shared_rss->ind_tbl) {
14471                 rte_flow_error_set(error, ENOMEM,
14472                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14473                                    "cannot allocate resource memory");
14474                 goto error_rss_init;
14475         }
14476         memcpy(queue, rss->queue, queue_size);
14477         shared_rss->ind_tbl->queues = queue;
14478         shared_rss->ind_tbl->queues_n = rss->queue_num;
14479         origin = &shared_rss->origin;
14480         origin->func = rss->func;
14481         origin->level = rss->level;
14482         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
14483         origin->types = !rss->types ? ETH_RSS_IP : rss->types;
14484         /* NULL RSS key indicates default RSS key. */
14485         rss_key = !rss->key ? rss_hash_default_key : rss->key;
14486         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
14487         origin->key = &shared_rss->key[0];
14488         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
14489         origin->queue = queue;
14490         origin->queue_num = rss->queue_num;
14491         if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error))
14492                 goto error_rss_init;
14493         rte_spinlock_init(&shared_rss->action_rss_sl);
14494         __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14495         rte_spinlock_lock(&priv->shared_act_sl);
14496         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14497                      &priv->rss_shared_actions, idx, shared_rss, next);
14498         rte_spinlock_unlock(&priv->shared_act_sl);
14499         return idx;
14500 error_rss_init:
14501         if (shared_rss) {
14502                 if (shared_rss->ind_tbl)
14503                         mlx5_free(shared_rss->ind_tbl);
14504                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14505                                 idx);
14506         }
14507         if (queue)
14508                 mlx5_free(queue);
14509         return 0;
14510 }
14511
14512 /**
14513  * Destroy the shared RSS action.
14514  * Release related hash RX queue objects.
14515  *
14516  * @param[in] dev
14517  *   Pointer to the Ethernet device structure.
14518  * @param[in] idx
14519  *   The shared RSS action object ID to be removed.
14520  * @param[out] error
14521  *   Perform verbose error reporting if not NULL. Initialized in case of
14522  *   error only.
14523  *
14524  * @return
14525  *   0 on success, otherwise negative errno value.
14526  */
14527 static int
14528 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx,
14529                              struct rte_flow_error *error)
14530 {
14531         struct mlx5_priv *priv = dev->data->dev_private;
14532         struct mlx5_shared_action_rss *shared_rss =
14533             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14534         uint32_t old_refcnt = 1;
14535         int remaining;
14536         uint16_t *queue = NULL;
14537
14538         if (!shared_rss)
14539                 return rte_flow_error_set(error, EINVAL,
14540                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14541                                           "invalid shared action");
14542         remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14543         if (remaining)
14544                 return rte_flow_error_set(error, EBUSY,
14545                                           RTE_FLOW_ERROR_TYPE_ACTION,
14546                                           NULL,
14547                                           "shared rss hrxq has references");
14548         if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt,
14549                                          0, 0, __ATOMIC_ACQUIRE,
14550                                          __ATOMIC_RELAXED))
14551                 return rte_flow_error_set(error, EBUSY,
14552                                           RTE_FLOW_ERROR_TYPE_ACTION,
14553                                           NULL,
14554                                           "shared rss has references");
14555         queue = shared_rss->ind_tbl->queues;
14556         remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true);
14557         if (remaining)
14558                 return rte_flow_error_set(error, EBUSY,
14559                                           RTE_FLOW_ERROR_TYPE_ACTION,
14560                                           NULL,
14561                                           "shared rss indirection table has"
14562                                           " references");
14563         mlx5_free(queue);
14564         rte_spinlock_lock(&priv->shared_act_sl);
14565         ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14566                      &priv->rss_shared_actions, idx, shared_rss, next);
14567         rte_spinlock_unlock(&priv->shared_act_sl);
14568         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14569                         idx);
14570         return 0;
14571 }
14572
14573 /**
14574  * Create indirect action, lock free,
14575  * (mutex should be acquired by caller).
14576  * Dispatcher for action type specific call.
14577  *
14578  * @param[in] dev
14579  *   Pointer to the Ethernet device structure.
14580  * @param[in] conf
14581  *   Shared action configuration.
14582  * @param[in] action
14583  *   Action specification used to create indirect action.
14584  * @param[out] error
14585  *   Perform verbose error reporting if not NULL. Initialized in case of
14586  *   error only.
14587  *
14588  * @return
14589  *   A valid shared action handle in case of success, NULL otherwise and
14590  *   rte_errno is set.
14591  */
14592 static struct rte_flow_action_handle *
14593 flow_dv_action_create(struct rte_eth_dev *dev,
14594                       const struct rte_flow_indir_action_conf *conf,
14595                       const struct rte_flow_action *action,
14596                       struct rte_flow_error *err)
14597 {
14598         struct mlx5_priv *priv = dev->data->dev_private;
14599         uint32_t age_idx = 0;
14600         uint32_t idx = 0;
14601         uint32_t ret = 0;
14602
14603         switch (action->type) {
14604         case RTE_FLOW_ACTION_TYPE_RSS:
14605                 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err);
14606                 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS <<
14607                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14608                 break;
14609         case RTE_FLOW_ACTION_TYPE_AGE:
14610                 age_idx = flow_dv_aso_age_alloc(dev, err);
14611                 if (!age_idx) {
14612                         ret = -rte_errno;
14613                         break;
14614                 }
14615                 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
14616                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
14617                 flow_dv_aso_age_params_init(dev, age_idx,
14618                                         ((const struct rte_flow_action_age *)
14619                                                 action->conf)->context ?
14620                                         ((const struct rte_flow_action_age *)
14621                                                 action->conf)->context :
14622                                         (void *)(uintptr_t)idx,
14623                                         ((const struct rte_flow_action_age *)
14624                                                 action->conf)->timeout);
14625                 ret = age_idx;
14626                 break;
14627         case RTE_FLOW_ACTION_TYPE_COUNT:
14628                 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL);
14629                 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT <<
14630                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14631                 break;
14632         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
14633                 ret = flow_dv_translate_create_conntrack(dev, action->conf,
14634                                                          err);
14635                 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret);
14636                 break;
14637         default:
14638                 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
14639                                    NULL, "action type not supported");
14640                 break;
14641         }
14642         return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL;
14643 }
14644
14645 /**
14646  * Destroy the indirect action.
14647  * Release action related resources on the NIC and the memory.
14648  * Lock free, (mutex should be acquired by caller).
14649  * Dispatcher for action type specific call.
14650  *
14651  * @param[in] dev
14652  *   Pointer to the Ethernet device structure.
14653  * @param[in] handle
14654  *   The indirect action object handle to be removed.
14655  * @param[out] error
14656  *   Perform verbose error reporting if not NULL. Initialized in case of
14657  *   error only.
14658  *
14659  * @return
14660  *   0 on success, otherwise negative errno value.
14661  */
14662 static int
14663 flow_dv_action_destroy(struct rte_eth_dev *dev,
14664                        struct rte_flow_action_handle *handle,
14665                        struct rte_flow_error *error)
14666 {
14667         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14668         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14669         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14670         struct mlx5_flow_counter *cnt;
14671         uint32_t no_flow_refcnt = 1;
14672         int ret;
14673
14674         switch (type) {
14675         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14676                 return __flow_dv_action_rss_release(dev, idx, error);
14677         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
14678                 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL);
14679                 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt,
14680                                                  &no_flow_refcnt, 1, false,
14681                                                  __ATOMIC_ACQUIRE,
14682                                                  __ATOMIC_RELAXED))
14683                         return rte_flow_error_set(error, EBUSY,
14684                                                   RTE_FLOW_ERROR_TYPE_ACTION,
14685                                                   NULL,
14686                                                   "Indirect count action has references");
14687                 flow_dv_counter_free(dev, idx);
14688                 return 0;
14689         case MLX5_INDIRECT_ACTION_TYPE_AGE:
14690                 ret = flow_dv_aso_age_release(dev, idx);
14691                 if (ret)
14692                         /*
14693                          * In this case, the last flow has a reference will
14694                          * actually release the age action.
14695                          */
14696                         DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was"
14697                                 " released with references %d.", idx, ret);
14698                 return 0;
14699         case MLX5_INDIRECT_ACTION_TYPE_CT:
14700                 ret = flow_dv_aso_ct_release(dev, idx);
14701                 if (ret < 0)
14702                         return ret;
14703                 if (ret > 0)
14704                         DRV_LOG(DEBUG, "Connection tracking object %u still "
14705                                 "has references %d.", idx, ret);
14706                 return 0;
14707         default:
14708                 return rte_flow_error_set(error, ENOTSUP,
14709                                           RTE_FLOW_ERROR_TYPE_ACTION,
14710                                           NULL,
14711                                           "action type not supported");
14712         }
14713 }
14714
14715 /**
14716  * Updates in place shared RSS action configuration.
14717  *
14718  * @param[in] dev
14719  *   Pointer to the Ethernet device structure.
14720  * @param[in] idx
14721  *   The shared RSS action object ID to be updated.
14722  * @param[in] action_conf
14723  *   RSS action specification used to modify *shared_rss*.
14724  * @param[out] error
14725  *   Perform verbose error reporting if not NULL. Initialized in case of
14726  *   error only.
14727  *
14728  * @return
14729  *   0 on success, otherwise negative errno value.
14730  * @note: currently only support update of RSS queues.
14731  */
14732 static int
14733 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx,
14734                             const struct rte_flow_action_rss *action_conf,
14735                             struct rte_flow_error *error)
14736 {
14737         struct mlx5_priv *priv = dev->data->dev_private;
14738         struct mlx5_shared_action_rss *shared_rss =
14739             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14740         int ret = 0;
14741         void *queue = NULL;
14742         uint16_t *queue_old = NULL;
14743         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
14744
14745         if (!shared_rss)
14746                 return rte_flow_error_set(error, EINVAL,
14747                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14748                                           "invalid shared action to update");
14749         if (priv->obj_ops.ind_table_modify == NULL)
14750                 return rte_flow_error_set(error, ENOTSUP,
14751                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14752                                           "cannot modify indirection table");
14753         queue = mlx5_malloc(MLX5_MEM_ZERO,
14754                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14755                             0, SOCKET_ID_ANY);
14756         if (!queue)
14757                 return rte_flow_error_set(error, ENOMEM,
14758                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14759                                           NULL,
14760                                           "cannot allocate resource memory");
14761         memcpy(queue, action_conf->queue, queue_size);
14762         MLX5_ASSERT(shared_rss->ind_tbl);
14763         rte_spinlock_lock(&shared_rss->action_rss_sl);
14764         queue_old = shared_rss->ind_tbl->queues;
14765         ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl,
14766                                         queue, action_conf->queue_num, true);
14767         if (ret) {
14768                 mlx5_free(queue);
14769                 ret = rte_flow_error_set(error, rte_errno,
14770                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14771                                           "cannot update indirection table");
14772         } else {
14773                 mlx5_free(queue_old);
14774                 shared_rss->origin.queue = queue;
14775                 shared_rss->origin.queue_num = action_conf->queue_num;
14776         }
14777         rte_spinlock_unlock(&shared_rss->action_rss_sl);
14778         return ret;
14779 }
14780
14781 /*
14782  * Updates in place conntrack context or direction.
14783  * Context update should be synchronized.
14784  *
14785  * @param[in] dev
14786  *   Pointer to the Ethernet device structure.
14787  * @param[in] idx
14788  *   The conntrack object ID to be updated.
14789  * @param[in] update
14790  *   Pointer to the structure of information to update.
14791  * @param[out] error
14792  *   Perform verbose error reporting if not NULL. Initialized in case of
14793  *   error only.
14794  *
14795  * @return
14796  *   0 on success, otherwise negative errno value.
14797  */
14798 static int
14799 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx,
14800                            const struct rte_flow_modify_conntrack *update,
14801                            struct rte_flow_error *error)
14802 {
14803         struct mlx5_priv *priv = dev->data->dev_private;
14804         struct mlx5_aso_ct_action *ct;
14805         const struct rte_flow_action_conntrack *new_prf;
14806         int ret = 0;
14807         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
14808         uint32_t dev_idx;
14809
14810         if (PORT_ID(priv) != owner)
14811                 return rte_flow_error_set(error, EACCES,
14812                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14813                                           NULL,
14814                                           "CT object owned by another port");
14815         dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
14816         ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
14817         if (!ct->refcnt)
14818                 return rte_flow_error_set(error, ENOMEM,
14819                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14820                                           NULL,
14821                                           "CT object is inactive");
14822         new_prf = &update->new_ct;
14823         if (update->direction)
14824                 ct->is_original = !!new_prf->is_original_dir;
14825         if (update->state) {
14826                 /* Only validate the profile when it needs to be updated. */
14827                 ret = mlx5_validate_action_ct(dev, new_prf, error);
14828                 if (ret)
14829                         return ret;
14830                 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf);
14831                 if (ret)
14832                         return rte_flow_error_set(error, EIO,
14833                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14834                                         NULL,
14835                                         "Failed to send CT context update WQE");
14836                 /* Block until ready or a failure. */
14837                 ret = mlx5_aso_ct_available(priv->sh, ct);
14838                 if (ret)
14839                         rte_flow_error_set(error, rte_errno,
14840                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14841                                            NULL,
14842                                            "Timeout to get the CT update");
14843         }
14844         return ret;
14845 }
14846
14847 /**
14848  * Updates in place shared action configuration, lock free,
14849  * (mutex should be acquired by caller).
14850  *
14851  * @param[in] dev
14852  *   Pointer to the Ethernet device structure.
14853  * @param[in] handle
14854  *   The indirect action object handle to be updated.
14855  * @param[in] update
14856  *   Action specification used to modify the action pointed by *handle*.
14857  *   *update* could be of same type with the action pointed by the *handle*
14858  *   handle argument, or some other structures like a wrapper, depending on
14859  *   the indirect action type.
14860  * @param[out] error
14861  *   Perform verbose error reporting if not NULL. Initialized in case of
14862  *   error only.
14863  *
14864  * @return
14865  *   0 on success, otherwise negative errno value.
14866  */
14867 static int
14868 flow_dv_action_update(struct rte_eth_dev *dev,
14869                         struct rte_flow_action_handle *handle,
14870                         const void *update,
14871                         struct rte_flow_error *err)
14872 {
14873         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14874         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14875         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14876         const void *action_conf;
14877
14878         switch (type) {
14879         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14880                 action_conf = ((const struct rte_flow_action *)update)->conf;
14881                 return __flow_dv_action_rss_update(dev, idx, action_conf, err);
14882         case MLX5_INDIRECT_ACTION_TYPE_CT:
14883                 return __flow_dv_action_ct_update(dev, idx, update, err);
14884         default:
14885                 return rte_flow_error_set(err, ENOTSUP,
14886                                           RTE_FLOW_ERROR_TYPE_ACTION,
14887                                           NULL,
14888                                           "action type update not supported");
14889         }
14890 }
14891
14892 /**
14893  * Destroy the meter sub policy table rules.
14894  * Lock free, (mutex should be acquired by caller).
14895  *
14896  * @param[in] dev
14897  *   Pointer to Ethernet device.
14898  * @param[in] sub_policy
14899  *   Pointer to meter sub policy table.
14900  */
14901 static void
14902 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
14903                              struct mlx5_flow_meter_sub_policy *sub_policy)
14904 {
14905         struct mlx5_priv *priv = dev->data->dev_private;
14906         struct mlx5_flow_tbl_data_entry *tbl;
14907         struct mlx5_flow_meter_policy *policy = sub_policy->main_policy;
14908         struct mlx5_flow_meter_info *next_fm;
14909         struct mlx5_sub_policy_color_rule *color_rule;
14910         void *tmp;
14911         uint32_t i;
14912
14913         for (i = 0; i < RTE_COLORS; i++) {
14914                 next_fm = NULL;
14915                 if (i == RTE_COLOR_GREEN && policy &&
14916                     policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
14917                         next_fm = mlx5_flow_meter_find(priv,
14918                                         policy->act_cnt[i].next_mtr_id, NULL);
14919                 TAILQ_FOREACH_SAFE(color_rule, &sub_policy->color_rules[i],
14920                                    next_port, tmp) {
14921                         claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
14922                         tbl = container_of(color_rule->matcher->tbl,
14923                                         typeof(*tbl), tbl);
14924                         mlx5_list_unregister(tbl->matchers,
14925                                                 &color_rule->matcher->entry);
14926                         TAILQ_REMOVE(&sub_policy->color_rules[i],
14927                                         color_rule, next_port);
14928                         mlx5_free(color_rule);
14929                         if (next_fm)
14930                                 mlx5_flow_meter_detach(priv, next_fm);
14931                 }
14932         }
14933         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
14934                 if (sub_policy->rix_hrxq[i]) {
14935                         if (policy && !policy->is_hierarchy)
14936                                 mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]);
14937                         sub_policy->rix_hrxq[i] = 0;
14938                 }
14939                 if (sub_policy->jump_tbl[i]) {
14940                         flow_dv_tbl_resource_release(MLX5_SH(dev),
14941                         sub_policy->jump_tbl[i]);
14942                         sub_policy->jump_tbl[i] = NULL;
14943                 }
14944         }
14945         if (sub_policy->tbl_rsc) {
14946                 flow_dv_tbl_resource_release(MLX5_SH(dev),
14947                         sub_policy->tbl_rsc);
14948                 sub_policy->tbl_rsc = NULL;
14949         }
14950 }
14951
14952 /**
14953  * Destroy policy rules, lock free,
14954  * (mutex should be acquired by caller).
14955  * Dispatcher for action type specific call.
14956  *
14957  * @param[in] dev
14958  *   Pointer to the Ethernet device structure.
14959  * @param[in] mtr_policy
14960  *   Meter policy struct.
14961  */
14962 static void
14963 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev,
14964                       struct mlx5_flow_meter_policy *mtr_policy)
14965 {
14966         uint32_t i, j;
14967         struct mlx5_flow_meter_sub_policy *sub_policy;
14968         uint16_t sub_policy_num;
14969
14970         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
14971                 sub_policy_num = (mtr_policy->sub_policy_num >>
14972                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
14973                         MLX5_MTR_SUB_POLICY_NUM_MASK;
14974                 for (j = 0; j < sub_policy_num; j++) {
14975                         sub_policy = mtr_policy->sub_policys[i][j];
14976                         if (sub_policy)
14977                                 __flow_dv_destroy_sub_policy_rules
14978                                                 (dev, sub_policy);
14979                 }
14980         }
14981 }
14982
14983 /**
14984  * Destroy policy action, lock free,
14985  * (mutex should be acquired by caller).
14986  * Dispatcher for action type specific call.
14987  *
14988  * @param[in] dev
14989  *   Pointer to the Ethernet device structure.
14990  * @param[in] mtr_policy
14991  *   Meter policy struct.
14992  */
14993 static void
14994 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
14995                       struct mlx5_flow_meter_policy *mtr_policy)
14996 {
14997         struct rte_flow_action *rss_action;
14998         struct mlx5_flow_handle dev_handle;
14999         uint32_t i, j;
15000
15001         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15002                 if (mtr_policy->act_cnt[i].rix_mark) {
15003                         flow_dv_tag_release(dev,
15004                                 mtr_policy->act_cnt[i].rix_mark);
15005                         mtr_policy->act_cnt[i].rix_mark = 0;
15006                 }
15007                 if (mtr_policy->act_cnt[i].modify_hdr) {
15008                         dev_handle.dvh.modify_hdr =
15009                                 mtr_policy->act_cnt[i].modify_hdr;
15010                         flow_dv_modify_hdr_resource_release(dev, &dev_handle);
15011                 }
15012                 switch (mtr_policy->act_cnt[i].fate_action) {
15013                 case MLX5_FLOW_FATE_SHARED_RSS:
15014                         rss_action = mtr_policy->act_cnt[i].rss;
15015                         mlx5_free(rss_action);
15016                         break;
15017                 case MLX5_FLOW_FATE_PORT_ID:
15018                         if (mtr_policy->act_cnt[i].rix_port_id_action) {
15019                                 flow_dv_port_id_action_resource_release(dev,
15020                                 mtr_policy->act_cnt[i].rix_port_id_action);
15021                                 mtr_policy->act_cnt[i].rix_port_id_action = 0;
15022                         }
15023                         break;
15024                 case MLX5_FLOW_FATE_DROP:
15025                 case MLX5_FLOW_FATE_JUMP:
15026                         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15027                                 mtr_policy->act_cnt[i].dr_jump_action[j] =
15028                                                 NULL;
15029                         break;
15030                 default:
15031                         /*Queue action do nothing*/
15032                         break;
15033                 }
15034         }
15035         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15036                 mtr_policy->dr_drop_action[j] = NULL;
15037 }
15038
15039 /**
15040  * Create policy action per domain, lock free,
15041  * (mutex should be acquired by caller).
15042  * Dispatcher for action type specific call.
15043  *
15044  * @param[in] dev
15045  *   Pointer to the Ethernet device structure.
15046  * @param[in] mtr_policy
15047  *   Meter policy struct.
15048  * @param[in] action
15049  *   Action specification used to create meter actions.
15050  * @param[out] error
15051  *   Perform verbose error reporting if not NULL. Initialized in case of
15052  *   error only.
15053  *
15054  * @return
15055  *   0 on success, otherwise negative errno value.
15056  */
15057 static int
15058 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
15059                         struct mlx5_flow_meter_policy *mtr_policy,
15060                         const struct rte_flow_action *actions[RTE_COLORS],
15061                         enum mlx5_meter_domain domain,
15062                         struct rte_mtr_error *error)
15063 {
15064         struct mlx5_priv *priv = dev->data->dev_private;
15065         struct rte_flow_error flow_err;
15066         const struct rte_flow_action *act;
15067         uint64_t action_flags = 0;
15068         struct mlx5_flow_handle dh;
15069         struct mlx5_flow dev_flow;
15070         struct mlx5_flow_dv_port_id_action_resource port_id_action;
15071         int i, ret;
15072         uint8_t egress, transfer;
15073         struct mlx5_meter_policy_action_container *act_cnt = NULL;
15074         union {
15075                 struct mlx5_flow_dv_modify_hdr_resource res;
15076                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
15077                             sizeof(struct mlx5_modification_cmd) *
15078                             (MLX5_MAX_MODIFY_NUM + 1)];
15079         } mhdr_dummy;
15080         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
15081
15082         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15083         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15084         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
15085         memset(&dev_flow, 0, sizeof(struct mlx5_flow));
15086         memset(&port_id_action, 0,
15087                 sizeof(struct mlx5_flow_dv_port_id_action_resource));
15088         memset(mhdr_res, 0, sizeof(*mhdr_res));
15089         mhdr_res->ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
15090                                         egress ?
15091                                         MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
15092                                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
15093         dev_flow.handle = &dh;
15094         dev_flow.dv.port_id_action = &port_id_action;
15095         dev_flow.external = true;
15096         for (i = 0; i < RTE_COLORS; i++) {
15097                 if (i < MLX5_MTR_RTE_COLORS)
15098                         act_cnt = &mtr_policy->act_cnt[i];
15099                 for (act = actions[i];
15100                         act && act->type != RTE_FLOW_ACTION_TYPE_END;
15101                         act++) {
15102                         switch (act->type) {
15103                         case RTE_FLOW_ACTION_TYPE_MARK:
15104                         {
15105                                 uint32_t tag_be = mlx5_flow_mark_set
15106                                         (((const struct rte_flow_action_mark *)
15107                                         (act->conf))->id);
15108
15109                                 if (i >= MLX5_MTR_RTE_COLORS)
15110                                         return -rte_mtr_error_set(error,
15111                                           ENOTSUP,
15112                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15113                                           NULL,
15114                                           "cannot create policy "
15115                                           "mark action for this color");
15116                                 dev_flow.handle->mark = 1;
15117                                 if (flow_dv_tag_resource_register(dev, tag_be,
15118                                                   &dev_flow, &flow_err))
15119                                         return -rte_mtr_error_set(error,
15120                                         ENOTSUP,
15121                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15122                                         NULL,
15123                                         "cannot setup policy mark action");
15124                                 MLX5_ASSERT(dev_flow.dv.tag_resource);
15125                                 act_cnt->rix_mark =
15126                                         dev_flow.handle->dvh.rix_tag;
15127                                 action_flags |= MLX5_FLOW_ACTION_MARK;
15128                                 break;
15129                         }
15130                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
15131                                 if (i >= MLX5_MTR_RTE_COLORS)
15132                                         return -rte_mtr_error_set(error,
15133                                           ENOTSUP,
15134                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15135                                           NULL,
15136                                           "cannot create policy "
15137                                           "set tag action for this color");
15138                                 if (flow_dv_convert_action_set_tag
15139                                 (dev, mhdr_res,
15140                                 (const struct rte_flow_action_set_tag *)
15141                                 act->conf,  &flow_err))
15142                                         return -rte_mtr_error_set(error,
15143                                         ENOTSUP,
15144                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15145                                         NULL, "cannot convert policy "
15146                                         "set tag action");
15147                                 if (!mhdr_res->actions_num)
15148                                         return -rte_mtr_error_set(error,
15149                                         ENOTSUP,
15150                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15151                                         NULL, "cannot find policy "
15152                                         "set tag action");
15153                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
15154                                 break;
15155                         case RTE_FLOW_ACTION_TYPE_DROP:
15156                         {
15157                                 struct mlx5_flow_mtr_mng *mtrmng =
15158                                                 priv->sh->mtrmng;
15159                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15160
15161                                 /*
15162                                  * Create the drop table with
15163                                  * METER DROP level.
15164                                  */
15165                                 if (!mtrmng->drop_tbl[domain]) {
15166                                         mtrmng->drop_tbl[domain] =
15167                                         flow_dv_tbl_resource_get(dev,
15168                                         MLX5_FLOW_TABLE_LEVEL_METER,
15169                                         egress, transfer, false, NULL, 0,
15170                                         0, MLX5_MTR_TABLE_ID_DROP, &flow_err);
15171                                         if (!mtrmng->drop_tbl[domain])
15172                                                 return -rte_mtr_error_set
15173                                         (error, ENOTSUP,
15174                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15175                                         NULL,
15176                                         "Failed to create meter drop table");
15177                                 }
15178                                 tbl_data = container_of
15179                                 (mtrmng->drop_tbl[domain],
15180                                 struct mlx5_flow_tbl_data_entry, tbl);
15181                                 if (i < MLX5_MTR_RTE_COLORS) {
15182                                         act_cnt->dr_jump_action[domain] =
15183                                                 tbl_data->jump.action;
15184                                         act_cnt->fate_action =
15185                                                 MLX5_FLOW_FATE_DROP;
15186                                 }
15187                                 if (i == RTE_COLOR_RED)
15188                                         mtr_policy->dr_drop_action[domain] =
15189                                                 tbl_data->jump.action;
15190                                 action_flags |= MLX5_FLOW_ACTION_DROP;
15191                                 break;
15192                         }
15193                         case RTE_FLOW_ACTION_TYPE_QUEUE:
15194                         {
15195                                 if (i >= MLX5_MTR_RTE_COLORS)
15196                                         return -rte_mtr_error_set(error,
15197                                         ENOTSUP,
15198                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15199                                         NULL, "cannot create policy "
15200                                         "fate queue for this color");
15201                                 act_cnt->queue =
15202                                 ((const struct rte_flow_action_queue *)
15203                                         (act->conf))->index;
15204                                 act_cnt->fate_action =
15205                                         MLX5_FLOW_FATE_QUEUE;
15206                                 dev_flow.handle->fate_action =
15207                                         MLX5_FLOW_FATE_QUEUE;
15208                                 mtr_policy->is_queue = 1;
15209                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
15210                                 break;
15211                         }
15212                         case RTE_FLOW_ACTION_TYPE_RSS:
15213                         {
15214                                 int rss_size;
15215
15216                                 if (i >= MLX5_MTR_RTE_COLORS)
15217                                         return -rte_mtr_error_set(error,
15218                                           ENOTSUP,
15219                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15220                                           NULL,
15221                                           "cannot create policy "
15222                                           "rss action for this color");
15223                                 /*
15224                                  * Save RSS conf into policy struct
15225                                  * for translate stage.
15226                                  */
15227                                 rss_size = (int)rte_flow_conv
15228                                         (RTE_FLOW_CONV_OP_ACTION,
15229                                         NULL, 0, act, &flow_err);
15230                                 if (rss_size <= 0)
15231                                         return -rte_mtr_error_set(error,
15232                                           ENOTSUP,
15233                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15234                                           NULL, "Get the wrong "
15235                                           "rss action struct size");
15236                                 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO,
15237                                                 rss_size, 0, SOCKET_ID_ANY);
15238                                 if (!act_cnt->rss)
15239                                         return -rte_mtr_error_set(error,
15240                                           ENOTSUP,
15241                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15242                                           NULL,
15243                                           "Fail to malloc rss action memory");
15244                                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION,
15245                                         act_cnt->rss, rss_size,
15246                                         act, &flow_err);
15247                                 if (ret < 0)
15248                                         return -rte_mtr_error_set(error,
15249                                           ENOTSUP,
15250                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15251                                           NULL, "Fail to save "
15252                                           "rss action into policy struct");
15253                                 act_cnt->fate_action =
15254                                         MLX5_FLOW_FATE_SHARED_RSS;
15255                                 action_flags |= MLX5_FLOW_ACTION_RSS;
15256                                 break;
15257                         }
15258                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
15259                         {
15260                                 struct mlx5_flow_dv_port_id_action_resource
15261                                         port_id_resource;
15262                                 uint32_t port_id = 0;
15263
15264                                 if (i >= MLX5_MTR_RTE_COLORS)
15265                                         return -rte_mtr_error_set(error,
15266                                         ENOTSUP,
15267                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15268                                         NULL, "cannot create policy "
15269                                         "port action for this color");
15270                                 memset(&port_id_resource, 0,
15271                                         sizeof(port_id_resource));
15272                                 if (flow_dv_translate_action_port_id(dev, act,
15273                                                 &port_id, &flow_err))
15274                                         return -rte_mtr_error_set(error,
15275                                         ENOTSUP,
15276                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15277                                         NULL, "cannot translate "
15278                                         "policy port action");
15279                                 port_id_resource.port_id = port_id;
15280                                 if (flow_dv_port_id_action_resource_register
15281                                         (dev, &port_id_resource,
15282                                         &dev_flow, &flow_err))
15283                                         return -rte_mtr_error_set(error,
15284                                         ENOTSUP,
15285                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15286                                         NULL, "cannot setup "
15287                                         "policy port action");
15288                                 act_cnt->rix_port_id_action =
15289                                         dev_flow.handle->rix_port_id_action;
15290                                 act_cnt->fate_action =
15291                                         MLX5_FLOW_FATE_PORT_ID;
15292                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
15293                                 break;
15294                         }
15295                         case RTE_FLOW_ACTION_TYPE_JUMP:
15296                         {
15297                                 uint32_t jump_group = 0;
15298                                 uint32_t table = 0;
15299                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15300                                 struct flow_grp_info grp_info = {
15301                                         .external = !!dev_flow.external,
15302                                         .transfer = !!transfer,
15303                                         .fdb_def_rule = !!priv->fdb_def_rule,
15304                                         .std_tbl_fix = 0,
15305                                         .skip_scale = dev_flow.skip_scale &
15306                                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
15307                                 };
15308                                 struct mlx5_flow_meter_sub_policy *sub_policy =
15309                                 mtr_policy->sub_policys[domain][0];
15310
15311                                 if (i >= MLX5_MTR_RTE_COLORS)
15312                                         return -rte_mtr_error_set(error,
15313                                           ENOTSUP,
15314                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15315                                           NULL,
15316                                           "cannot create policy "
15317                                           "jump action for this color");
15318                                 jump_group =
15319                                 ((const struct rte_flow_action_jump *)
15320                                                         act->conf)->group;
15321                                 if (mlx5_flow_group_to_table(dev, NULL,
15322                                                        jump_group,
15323                                                        &table,
15324                                                        &grp_info, &flow_err))
15325                                         return -rte_mtr_error_set(error,
15326                                         ENOTSUP,
15327                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15328                                         NULL, "cannot setup "
15329                                         "policy jump action");
15330                                 sub_policy->jump_tbl[i] =
15331                                 flow_dv_tbl_resource_get(dev,
15332                                         table, egress,
15333                                         transfer,
15334                                         !!dev_flow.external,
15335                                         NULL, jump_group, 0,
15336                                         0, &flow_err);
15337                                 if
15338                                 (!sub_policy->jump_tbl[i])
15339                                         return  -rte_mtr_error_set(error,
15340                                         ENOTSUP,
15341                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15342                                         NULL, "cannot create jump action.");
15343                                 tbl_data = container_of
15344                                 (sub_policy->jump_tbl[i],
15345                                 struct mlx5_flow_tbl_data_entry, tbl);
15346                                 act_cnt->dr_jump_action[domain] =
15347                                         tbl_data->jump.action;
15348                                 act_cnt->fate_action =
15349                                         MLX5_FLOW_FATE_JUMP;
15350                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
15351                                 break;
15352                         }
15353                         case RTE_FLOW_ACTION_TYPE_METER:
15354                         {
15355                                 const struct rte_flow_action_meter *mtr;
15356                                 struct mlx5_flow_meter_info *next_fm;
15357                                 struct mlx5_flow_meter_policy *next_policy;
15358                                 struct rte_flow_action tag_action;
15359                                 struct mlx5_rte_flow_action_set_tag set_tag;
15360                                 uint32_t next_mtr_idx = 0;
15361
15362                                 mtr = act->conf;
15363                                 next_fm = mlx5_flow_meter_find(priv,
15364                                                         mtr->mtr_id,
15365                                                         &next_mtr_idx);
15366                                 if (!next_fm)
15367                                         return -rte_mtr_error_set(error, EINVAL,
15368                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15369                                                 "Fail to find next meter.");
15370                                 if (next_fm->def_policy)
15371                                         return -rte_mtr_error_set(error, EINVAL,
15372                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15373                                 "Hierarchy only supports termination meter.");
15374                                 next_policy = mlx5_flow_meter_policy_find(dev,
15375                                                 next_fm->policy_id, NULL);
15376                                 MLX5_ASSERT(next_policy);
15377                                 if (next_fm->drop_cnt) {
15378                                         set_tag.id =
15379                                                 (enum modify_reg)
15380                                                 mlx5_flow_get_reg_id(dev,
15381                                                 MLX5_MTR_ID,
15382                                                 0,
15383                                                 (struct rte_flow_error *)error);
15384                                         set_tag.offset = (priv->mtr_reg_share ?
15385                                                 MLX5_MTR_COLOR_BITS : 0);
15386                                         set_tag.length = (priv->mtr_reg_share ?
15387                                                MLX5_MTR_IDLE_BITS_IN_COLOR_REG :
15388                                                MLX5_REG_BITS);
15389                                         set_tag.data = next_mtr_idx;
15390                                         tag_action.type =
15391                                                 (enum rte_flow_action_type)
15392                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
15393                                         tag_action.conf = &set_tag;
15394                                         if (flow_dv_convert_action_set_reg
15395                                                 (mhdr_res, &tag_action,
15396                                                 (struct rte_flow_error *)error))
15397                                                 return -rte_errno;
15398                                         action_flags |=
15399                                                 MLX5_FLOW_ACTION_SET_TAG;
15400                                 }
15401                                 act_cnt->fate_action = MLX5_FLOW_FATE_MTR;
15402                                 act_cnt->next_mtr_id = next_fm->meter_id;
15403                                 act_cnt->next_sub_policy = NULL;
15404                                 mtr_policy->is_hierarchy = 1;
15405                                 mtr_policy->dev = next_policy->dev;
15406                                 action_flags |=
15407                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
15408                                 break;
15409                         }
15410                         default:
15411                                 return -rte_mtr_error_set(error, ENOTSUP,
15412                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15413                                           NULL, "action type not supported");
15414                         }
15415                         if (action_flags & MLX5_FLOW_ACTION_SET_TAG) {
15416                                 /* create modify action if needed. */
15417                                 dev_flow.dv.group = 1;
15418                                 if (flow_dv_modify_hdr_resource_register
15419                                         (dev, mhdr_res, &dev_flow, &flow_err))
15420                                         return -rte_mtr_error_set(error,
15421                                                 ENOTSUP,
15422                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
15423                                                 NULL, "cannot register policy "
15424                                                 "set tag action");
15425                                 act_cnt->modify_hdr =
15426                                         dev_flow.handle->dvh.modify_hdr;
15427                         }
15428                 }
15429         }
15430         return 0;
15431 }
15432
15433 /**
15434  * Create policy action per domain, lock free,
15435  * (mutex should be acquired by caller).
15436  * Dispatcher for action type specific call.
15437  *
15438  * @param[in] dev
15439  *   Pointer to the Ethernet device structure.
15440  * @param[in] mtr_policy
15441  *   Meter policy struct.
15442  * @param[in] action
15443  *   Action specification used to create meter actions.
15444  * @param[out] error
15445  *   Perform verbose error reporting if not NULL. Initialized in case of
15446  *   error only.
15447  *
15448  * @return
15449  *   0 on success, otherwise negative errno value.
15450  */
15451 static int
15452 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev,
15453                       struct mlx5_flow_meter_policy *mtr_policy,
15454                       const struct rte_flow_action *actions[RTE_COLORS],
15455                       struct rte_mtr_error *error)
15456 {
15457         int ret, i;
15458         uint16_t sub_policy_num;
15459
15460         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15461                 sub_policy_num = (mtr_policy->sub_policy_num >>
15462                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15463                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15464                 if (sub_policy_num) {
15465                         ret = __flow_dv_create_domain_policy_acts(dev,
15466                                 mtr_policy, actions,
15467                                 (enum mlx5_meter_domain)i, error);
15468                         if (ret)
15469                                 return ret;
15470                 }
15471         }
15472         return 0;
15473 }
15474
15475 /**
15476  * Query a DV flow rule for its statistics via DevX.
15477  *
15478  * @param[in] dev
15479  *   Pointer to Ethernet device.
15480  * @param[in] cnt_idx
15481  *   Index to the flow counter.
15482  * @param[out] data
15483  *   Data retrieved by the query.
15484  * @param[out] error
15485  *   Perform verbose error reporting if not NULL.
15486  *
15487  * @return
15488  *   0 on success, a negative errno value otherwise and rte_errno is set.
15489  */
15490 static int
15491 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data,
15492                     struct rte_flow_error *error)
15493 {
15494         struct mlx5_priv *priv = dev->data->dev_private;
15495         struct rte_flow_query_count *qc = data;
15496
15497         if (!priv->config.devx)
15498                 return rte_flow_error_set(error, ENOTSUP,
15499                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15500                                           NULL,
15501                                           "counters are not supported");
15502         if (cnt_idx) {
15503                 uint64_t pkts, bytes;
15504                 struct mlx5_flow_counter *cnt;
15505                 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes);
15506
15507                 if (err)
15508                         return rte_flow_error_set(error, -err,
15509                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15510                                         NULL, "cannot read counters");
15511                 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL);
15512                 qc->hits_set = 1;
15513                 qc->bytes_set = 1;
15514                 qc->hits = pkts - cnt->hits;
15515                 qc->bytes = bytes - cnt->bytes;
15516                 if (qc->reset) {
15517                         cnt->hits = pkts;
15518                         cnt->bytes = bytes;
15519                 }
15520                 return 0;
15521         }
15522         return rte_flow_error_set(error, EINVAL,
15523                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15524                                   NULL,
15525                                   "counters are not available");
15526 }
15527
15528 static int
15529 flow_dv_action_query(struct rte_eth_dev *dev,
15530                      const struct rte_flow_action_handle *handle, void *data,
15531                      struct rte_flow_error *error)
15532 {
15533         struct mlx5_age_param *age_param;
15534         struct rte_flow_query_age *resp;
15535         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15536         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15537         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15538         struct mlx5_priv *priv = dev->data->dev_private;
15539         struct mlx5_aso_ct_action *ct;
15540         uint16_t owner;
15541         uint32_t dev_idx;
15542
15543         switch (type) {
15544         case MLX5_INDIRECT_ACTION_TYPE_AGE:
15545                 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params;
15546                 resp = data;
15547                 resp->aged = __atomic_load_n(&age_param->state,
15548                                               __ATOMIC_RELAXED) == AGE_TMOUT ?
15549                                                                           1 : 0;
15550                 resp->sec_since_last_hit_valid = !resp->aged;
15551                 if (resp->sec_since_last_hit_valid)
15552                         resp->sec_since_last_hit = __atomic_load_n
15553                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15554                 return 0;
15555         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
15556                 return flow_dv_query_count(dev, idx, data, error);
15557         case MLX5_INDIRECT_ACTION_TYPE_CT:
15558                 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15559                 if (owner != PORT_ID(priv))
15560                         return rte_flow_error_set(error, EACCES,
15561                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15562                                         NULL,
15563                                         "CT object owned by another port");
15564                 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15565                 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15566                 MLX5_ASSERT(ct);
15567                 if (!ct->refcnt)
15568                         return rte_flow_error_set(error, EFAULT,
15569                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15570                                         NULL,
15571                                         "CT object is inactive");
15572                 ((struct rte_flow_action_conntrack *)data)->peer_port =
15573                                                         ct->peer;
15574                 ((struct rte_flow_action_conntrack *)data)->is_original_dir =
15575                                                         ct->is_original;
15576                 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data))
15577                         return rte_flow_error_set(error, EIO,
15578                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15579                                         NULL,
15580                                         "Failed to query CT context");
15581                 return 0;
15582         default:
15583                 return rte_flow_error_set(error, ENOTSUP,
15584                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15585                                           "action type query not supported");
15586         }
15587 }
15588
15589 /**
15590  * Query a flow rule AGE action for aging information.
15591  *
15592  * @param[in] dev
15593  *   Pointer to Ethernet device.
15594  * @param[in] flow
15595  *   Pointer to the sub flow.
15596  * @param[out] data
15597  *   data retrieved by the query.
15598  * @param[out] error
15599  *   Perform verbose error reporting if not NULL.
15600  *
15601  * @return
15602  *   0 on success, a negative errno value otherwise and rte_errno is set.
15603  */
15604 static int
15605 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
15606                   void *data, struct rte_flow_error *error)
15607 {
15608         struct rte_flow_query_age *resp = data;
15609         struct mlx5_age_param *age_param;
15610
15611         if (flow->age) {
15612                 struct mlx5_aso_age_action *act =
15613                                      flow_aso_age_get_by_idx(dev, flow->age);
15614
15615                 age_param = &act->age_params;
15616         } else if (flow->counter) {
15617                 age_param = flow_dv_counter_idx_get_age(dev, flow->counter);
15618
15619                 if (!age_param || !age_param->timeout)
15620                         return rte_flow_error_set
15621                                         (error, EINVAL,
15622                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15623                                          NULL, "cannot read age data");
15624         } else {
15625                 return rte_flow_error_set(error, EINVAL,
15626                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15627                                           NULL, "age data not available");
15628         }
15629         resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
15630                                      AGE_TMOUT ? 1 : 0;
15631         resp->sec_since_last_hit_valid = !resp->aged;
15632         if (resp->sec_since_last_hit_valid)
15633                 resp->sec_since_last_hit = __atomic_load_n
15634                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15635         return 0;
15636 }
15637
15638 /**
15639  * Query a flow.
15640  *
15641  * @see rte_flow_query()
15642  * @see rte_flow_ops
15643  */
15644 static int
15645 flow_dv_query(struct rte_eth_dev *dev,
15646               struct rte_flow *flow __rte_unused,
15647               const struct rte_flow_action *actions __rte_unused,
15648               void *data __rte_unused,
15649               struct rte_flow_error *error __rte_unused)
15650 {
15651         int ret = -EINVAL;
15652
15653         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
15654                 switch (actions->type) {
15655                 case RTE_FLOW_ACTION_TYPE_VOID:
15656                         break;
15657                 case RTE_FLOW_ACTION_TYPE_COUNT:
15658                         ret = flow_dv_query_count(dev, flow->counter, data,
15659                                                   error);
15660                         break;
15661                 case RTE_FLOW_ACTION_TYPE_AGE:
15662                         ret = flow_dv_query_age(dev, flow, data, error);
15663                         break;
15664                 default:
15665                         return rte_flow_error_set(error, ENOTSUP,
15666                                                   RTE_FLOW_ERROR_TYPE_ACTION,
15667                                                   actions,
15668                                                   "action not supported");
15669                 }
15670         }
15671         return ret;
15672 }
15673
15674 /**
15675  * Destroy the meter table set.
15676  * Lock free, (mutex should be acquired by caller).
15677  *
15678  * @param[in] dev
15679  *   Pointer to Ethernet device.
15680  * @param[in] fm
15681  *   Meter information table.
15682  */
15683 static void
15684 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
15685                         struct mlx5_flow_meter_info *fm)
15686 {
15687         struct mlx5_priv *priv = dev->data->dev_private;
15688         int i;
15689
15690         if (!fm || !priv->config.dv_flow_en)
15691                 return;
15692         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15693                 if (fm->drop_rule[i]) {
15694                         claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
15695                         fm->drop_rule[i] = NULL;
15696                 }
15697         }
15698 }
15699
15700 static void
15701 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
15702 {
15703         struct mlx5_priv *priv = dev->data->dev_private;
15704         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15705         struct mlx5_flow_tbl_data_entry *tbl;
15706         int i, j;
15707
15708         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15709                 if (mtrmng->def_rule[i]) {
15710                         claim_zero(mlx5_flow_os_destroy_flow
15711                                         (mtrmng->def_rule[i]));
15712                         mtrmng->def_rule[i] = NULL;
15713                 }
15714                 if (mtrmng->def_matcher[i]) {
15715                         tbl = container_of(mtrmng->def_matcher[i]->tbl,
15716                                 struct mlx5_flow_tbl_data_entry, tbl);
15717                         mlx5_list_unregister(tbl->matchers,
15718                                              &mtrmng->def_matcher[i]->entry);
15719                         mtrmng->def_matcher[i] = NULL;
15720                 }
15721                 for (j = 0; j < MLX5_REG_BITS; j++) {
15722                         if (mtrmng->drop_matcher[i][j]) {
15723                                 tbl =
15724                                 container_of(mtrmng->drop_matcher[i][j]->tbl,
15725                                              struct mlx5_flow_tbl_data_entry,
15726                                              tbl);
15727                                 mlx5_list_unregister(tbl->matchers,
15728                                             &mtrmng->drop_matcher[i][j]->entry);
15729                                 mtrmng->drop_matcher[i][j] = NULL;
15730                         }
15731                 }
15732                 if (mtrmng->drop_tbl[i]) {
15733                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15734                                 mtrmng->drop_tbl[i]);
15735                         mtrmng->drop_tbl[i] = NULL;
15736                 }
15737         }
15738 }
15739
15740 /* Number of meter flow actions, count and jump or count and drop. */
15741 #define METER_ACTIONS 2
15742
15743 static void
15744 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
15745                               enum mlx5_meter_domain domain)
15746 {
15747         struct mlx5_priv *priv = dev->data->dev_private;
15748         struct mlx5_flow_meter_def_policy *def_policy =
15749                         priv->sh->mtrmng->def_policy[domain];
15750
15751         __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
15752         mlx5_free(def_policy);
15753         priv->sh->mtrmng->def_policy[domain] = NULL;
15754 }
15755
15756 /**
15757  * Destroy the default policy table set.
15758  *
15759  * @param[in] dev
15760  *   Pointer to Ethernet device.
15761  */
15762 static void
15763 flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
15764 {
15765         struct mlx5_priv *priv = dev->data->dev_private;
15766         int i;
15767
15768         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
15769                 if (priv->sh->mtrmng->def_policy[i])
15770                         __flow_dv_destroy_domain_def_policy(dev,
15771                                         (enum mlx5_meter_domain)i);
15772         priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
15773 }
15774
15775 static int
15776 __flow_dv_create_policy_flow(struct rte_eth_dev *dev,
15777                         uint32_t color_reg_c_idx,
15778                         enum rte_color color, void *matcher_object,
15779                         int actions_n, void *actions,
15780                         bool match_src_port, const struct rte_flow_item *item,
15781                         void **rule, const struct rte_flow_attr *attr)
15782 {
15783         int ret;
15784         struct mlx5_flow_dv_match_params value = {
15785                 .size = sizeof(value.buf),
15786         };
15787         struct mlx5_flow_dv_match_params matcher = {
15788                 .size = sizeof(matcher.buf),
15789         };
15790         struct mlx5_priv *priv = dev->data->dev_private;
15791         uint8_t misc_mask;
15792
15793         if (match_src_port && (priv->representor || priv->master)) {
15794                 if (flow_dv_translate_item_port_id(dev, matcher.buf,
15795                                                    value.buf, item, attr)) {
15796                         DRV_LOG(ERR,
15797                         "Failed to create meter policy flow with port.");
15798                         return -1;
15799                 }
15800         }
15801         flow_dv_match_meta_reg(matcher.buf, value.buf,
15802                                 (enum modify_reg)color_reg_c_idx,
15803                                 rte_col_2_mlx5_col(color),
15804                                 UINT32_MAX);
15805         misc_mask = flow_dv_matcher_enable(value.buf);
15806         __flow_dv_adjust_buf_size(&value.size, misc_mask);
15807         ret = mlx5_flow_os_create_flow(matcher_object,
15808                         (void *)&value, actions_n, actions, rule);
15809         if (ret) {
15810                 DRV_LOG(ERR, "Failed to create meter policy flow.");
15811                 return -1;
15812         }
15813         return 0;
15814 }
15815
15816 static int
15817 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
15818                         uint32_t color_reg_c_idx,
15819                         uint16_t priority,
15820                         struct mlx5_flow_meter_sub_policy *sub_policy,
15821                         const struct rte_flow_attr *attr,
15822                         bool match_src_port,
15823                         const struct rte_flow_item *item,
15824                         struct mlx5_flow_dv_matcher **policy_matcher,
15825                         struct rte_flow_error *error)
15826 {
15827         struct mlx5_list_entry *entry;
15828         struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
15829         struct mlx5_flow_dv_matcher matcher = {
15830                 .mask = {
15831                         .size = sizeof(matcher.mask.buf),
15832                 },
15833                 .tbl = tbl_rsc,
15834         };
15835         struct mlx5_flow_dv_match_params value = {
15836                 .size = sizeof(value.buf),
15837         };
15838         struct mlx5_flow_cb_ctx ctx = {
15839                 .error = error,
15840                 .data = &matcher,
15841         };
15842         struct mlx5_flow_tbl_data_entry *tbl_data;
15843         struct mlx5_priv *priv = dev->data->dev_private;
15844         uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
15845
15846         if (match_src_port && (priv->representor || priv->master)) {
15847                 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
15848                                                    value.buf, item, attr)) {
15849                         DRV_LOG(ERR,
15850                         "Failed to register meter drop matcher with port.");
15851                         return -1;
15852                 }
15853         }
15854         tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
15855         if (priority < RTE_COLOR_RED)
15856                 flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
15857                         (enum modify_reg)color_reg_c_idx, 0, color_mask);
15858         matcher.priority = priority;
15859         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
15860                                         matcher.mask.size);
15861         entry = mlx5_list_register(tbl_data->matchers, &ctx);
15862         if (!entry) {
15863                 DRV_LOG(ERR, "Failed to register meter drop matcher.");
15864                 return -1;
15865         }
15866         *policy_matcher =
15867                 container_of(entry, struct mlx5_flow_dv_matcher, entry);
15868         return 0;
15869 }
15870
15871 /**
15872  * Create the policy rules per domain.
15873  *
15874  * @param[in] dev
15875  *   Pointer to Ethernet device.
15876  * @param[in] sub_policy
15877  *    Pointer to sub policy table..
15878  * @param[in] egress
15879  *   Direction of the table.
15880  * @param[in] transfer
15881  *   E-Switch or NIC flow.
15882  * @param[in] acts
15883  *   Pointer to policy action list per color.
15884  *
15885  * @return
15886  *   0 on success, -1 otherwise.
15887  */
15888 static int
15889 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
15890                 struct mlx5_flow_meter_sub_policy *sub_policy,
15891                 uint8_t egress, uint8_t transfer, bool match_src_port,
15892                 struct mlx5_meter_policy_acts acts[RTE_COLORS])
15893 {
15894         struct mlx5_priv *priv = dev->data->dev_private;
15895         struct rte_flow_error flow_err;
15896         uint32_t color_reg_c_idx;
15897         struct rte_flow_attr attr = {
15898                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
15899                 .priority = 0,
15900                 .ingress = 0,
15901                 .egress = !!egress,
15902                 .transfer = !!transfer,
15903                 .reserved = 0,
15904         };
15905         int i;
15906         int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
15907         struct mlx5_sub_policy_color_rule *color_rule;
15908
15909         if (ret < 0)
15910                 return -1;
15911         /* Create policy table with POLICY level. */
15912         if (!sub_policy->tbl_rsc)
15913                 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
15914                                 MLX5_FLOW_TABLE_LEVEL_POLICY,
15915                                 egress, transfer, false, NULL, 0, 0,
15916                                 sub_policy->idx, &flow_err);
15917         if (!sub_policy->tbl_rsc) {
15918                 DRV_LOG(ERR,
15919                         "Failed to create meter sub policy table.");
15920                 return -1;
15921         }
15922         /* Prepare matchers. */
15923         color_reg_c_idx = ret;
15924         for (i = 0; i < RTE_COLORS; i++) {
15925                 TAILQ_INIT(&sub_policy->color_rules[i]);
15926                 if (i == RTE_COLOR_YELLOW || !acts[i].actions_n)
15927                         continue;
15928                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
15929                                 sizeof(struct mlx5_sub_policy_color_rule),
15930                                 0, SOCKET_ID_ANY);
15931                 if (!color_rule) {
15932                         DRV_LOG(ERR, "No memory to create color rule.");
15933                         goto err_exit;
15934                 }
15935                 color_rule->src_port = priv->representor_id;
15936                 attr.priority = i;
15937                 /* Create matchers for Color. */
15938                 if (__flow_dv_create_policy_matcher(dev,
15939                                 color_reg_c_idx, i, sub_policy, &attr,
15940                                 (i != RTE_COLOR_RED ? match_src_port : false),
15941                                 NULL, &color_rule->matcher, &flow_err)) {
15942                         DRV_LOG(ERR, "Failed to create color matcher.");
15943                         goto err_exit;
15944                 }
15945                 /* Create flow, matching color. */
15946                 if (__flow_dv_create_policy_flow(dev,
15947                                 color_reg_c_idx, (enum rte_color)i,
15948                                 color_rule->matcher->matcher_object,
15949                                 acts[i].actions_n,
15950                                 acts[i].dv_actions,
15951                                 (i != RTE_COLOR_RED ? match_src_port : false),
15952                                 NULL, &color_rule->rule,
15953                                 &attr)) {
15954                         DRV_LOG(ERR, "Failed to create color rule.");
15955                         goto err_exit;
15956                 }
15957                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
15958                                   color_rule, next_port);
15959         }
15960         return 0;
15961 err_exit:
15962         if (color_rule) {
15963                 if (color_rule->rule)
15964                         mlx5_flow_os_destroy_flow(color_rule->rule);
15965                 if (color_rule->matcher) {
15966                         struct mlx5_flow_tbl_data_entry *tbl =
15967                                 container_of(color_rule->matcher->tbl,
15968                                                 typeof(*tbl), tbl);
15969                         mlx5_list_unregister(tbl->matchers,
15970                                                 &color_rule->matcher->entry);
15971                 }
15972                 mlx5_free(color_rule);
15973         }
15974         return -1;
15975 }
15976
15977 static int
15978 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
15979                         struct mlx5_flow_meter_policy *mtr_policy,
15980                         struct mlx5_flow_meter_sub_policy *sub_policy,
15981                         uint32_t domain)
15982 {
15983         struct mlx5_priv *priv = dev->data->dev_private;
15984         struct mlx5_meter_policy_acts acts[RTE_COLORS];
15985         struct mlx5_flow_dv_tag_resource *tag;
15986         struct mlx5_flow_dv_port_id_action_resource *port_action;
15987         struct mlx5_hrxq *hrxq;
15988         struct mlx5_flow_meter_info *next_fm = NULL;
15989         struct mlx5_flow_meter_policy *next_policy;
15990         struct mlx5_flow_meter_sub_policy *next_sub_policy;
15991         struct mlx5_flow_tbl_data_entry *tbl_data;
15992         struct rte_flow_error error;
15993         uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15994         uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15995         bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX);
15996         bool match_src_port = false;
15997         int i;
15998
15999         for (i = 0; i < RTE_COLORS; i++) {
16000                 acts[i].actions_n = 0;
16001                 if (i == RTE_COLOR_YELLOW)
16002                         continue;
16003                 if (i == RTE_COLOR_RED) {
16004                         /* Only support drop on red. */
16005                         acts[i].dv_actions[0] =
16006                         mtr_policy->dr_drop_action[domain];
16007                         acts[i].actions_n = 1;
16008                         continue;
16009                 }
16010                 if (mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
16011                         struct rte_flow_attr attr = {
16012                                 .transfer = transfer
16013                         };
16014
16015                         next_fm = mlx5_flow_meter_find(priv,
16016                                         mtr_policy->act_cnt[i].next_mtr_id,
16017                                         NULL);
16018                         if (!next_fm) {
16019                                 DRV_LOG(ERR,
16020                                         "Failed to get next hierarchy meter.");
16021                                 goto err_exit;
16022                         }
16023                         if (mlx5_flow_meter_attach(priv, next_fm,
16024                                                    &attr, &error)) {
16025                                 DRV_LOG(ERR, "%s", error.message);
16026                                 next_fm = NULL;
16027                                 goto err_exit;
16028                         }
16029                         /* Meter action must be the first for TX. */
16030                         if (mtr_first) {
16031                                 acts[i].dv_actions[acts[i].actions_n] =
16032                                         next_fm->meter_action;
16033                                 acts[i].actions_n++;
16034                         }
16035                 }
16036                 if (mtr_policy->act_cnt[i].rix_mark) {
16037                         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
16038                                         mtr_policy->act_cnt[i].rix_mark);
16039                         if (!tag) {
16040                                 DRV_LOG(ERR, "Failed to find "
16041                                 "mark action for policy.");
16042                                 goto err_exit;
16043                         }
16044                         acts[i].dv_actions[acts[i].actions_n] =
16045                                                 tag->action;
16046                         acts[i].actions_n++;
16047                 }
16048                 if (mtr_policy->act_cnt[i].modify_hdr) {
16049                         acts[i].dv_actions[acts[i].actions_n] =
16050                         mtr_policy->act_cnt[i].modify_hdr->action;
16051                         acts[i].actions_n++;
16052                 }
16053                 if (mtr_policy->act_cnt[i].fate_action) {
16054                         switch (mtr_policy->act_cnt[i].fate_action) {
16055                         case MLX5_FLOW_FATE_PORT_ID:
16056                                 port_action = mlx5_ipool_get
16057                                         (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
16058                                 mtr_policy->act_cnt[i].rix_port_id_action);
16059                                 if (!port_action) {
16060                                         DRV_LOG(ERR, "Failed to find "
16061                                                 "port action for policy.");
16062                                         goto err_exit;
16063                                 }
16064                                 acts[i].dv_actions[acts[i].actions_n] =
16065                                 port_action->action;
16066                                 acts[i].actions_n++;
16067                                 mtr_policy->dev = dev;
16068                                 match_src_port = true;
16069                                 break;
16070                         case MLX5_FLOW_FATE_DROP:
16071                         case MLX5_FLOW_FATE_JUMP:
16072                                 acts[i].dv_actions[acts[i].actions_n] =
16073                                 mtr_policy->act_cnt[i].dr_jump_action[domain];
16074                                 acts[i].actions_n++;
16075                                 break;
16076                         case MLX5_FLOW_FATE_SHARED_RSS:
16077                         case MLX5_FLOW_FATE_QUEUE:
16078                                 hrxq = mlx5_ipool_get
16079                                 (priv->sh->ipool[MLX5_IPOOL_HRXQ],
16080                                 sub_policy->rix_hrxq[i]);
16081                                 if (!hrxq) {
16082                                         DRV_LOG(ERR, "Failed to find "
16083                                                 "queue action for policy.");
16084                                         goto err_exit;
16085                                 }
16086                                 acts[i].dv_actions[acts[i].actions_n] =
16087                                 hrxq->action;
16088                                 acts[i].actions_n++;
16089                                 break;
16090                         case MLX5_FLOW_FATE_MTR:
16091                                 if (!next_fm) {
16092                                         DRV_LOG(ERR,
16093                                                 "No next hierarchy meter.");
16094                                         goto err_exit;
16095                                 }
16096                                 if (!mtr_first) {
16097                                         acts[i].dv_actions[acts[i].actions_n] =
16098                                                         next_fm->meter_action;
16099                                         acts[i].actions_n++;
16100                                 }
16101                                 if (mtr_policy->act_cnt[i].next_sub_policy) {
16102                                         next_sub_policy =
16103                                         mtr_policy->act_cnt[i].next_sub_policy;
16104                                 } else {
16105                                         next_policy =
16106                                                 mlx5_flow_meter_policy_find(dev,
16107                                                 next_fm->policy_id, NULL);
16108                                         MLX5_ASSERT(next_policy);
16109                                         next_sub_policy =
16110                                         next_policy->sub_policys[domain][0];
16111                                 }
16112                                 tbl_data =
16113                                         container_of(next_sub_policy->tbl_rsc,
16114                                         struct mlx5_flow_tbl_data_entry, tbl);
16115                                 acts[i].dv_actions[acts[i].actions_n++] =
16116                                                         tbl_data->jump.action;
16117                                 if (mtr_policy->act_cnt[i].modify_hdr)
16118                                         match_src_port = !!transfer;
16119                                 break;
16120                         default:
16121                                 /*Queue action do nothing*/
16122                                 break;
16123                         }
16124                 }
16125         }
16126         if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
16127                                 egress, transfer, match_src_port, acts)) {
16128                 DRV_LOG(ERR,
16129                 "Failed to create policy rules per domain.");
16130                 goto err_exit;
16131         }
16132         return 0;
16133 err_exit:
16134         if (next_fm)
16135                 mlx5_flow_meter_detach(priv, next_fm);
16136         return -1;
16137 }
16138
16139 /**
16140  * Create the policy rules.
16141  *
16142  * @param[in] dev
16143  *   Pointer to Ethernet device.
16144  * @param[in,out] mtr_policy
16145  *   Pointer to meter policy table.
16146  *
16147  * @return
16148  *   0 on success, -1 otherwise.
16149  */
16150 static int
16151 flow_dv_create_policy_rules(struct rte_eth_dev *dev,
16152                              struct mlx5_flow_meter_policy *mtr_policy)
16153 {
16154         int i;
16155         uint16_t sub_policy_num;
16156
16157         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16158                 sub_policy_num = (mtr_policy->sub_policy_num >>
16159                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
16160                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16161                 if (!sub_policy_num)
16162                         continue;
16163                 /* Prepare actions list and create policy rules. */
16164                 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16165                         mtr_policy->sub_policys[i][0], i)) {
16166                         DRV_LOG(ERR,
16167                         "Failed to create policy action list per domain.");
16168                         return -1;
16169                 }
16170         }
16171         return 0;
16172 }
16173
16174 static int
16175 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
16176 {
16177         struct mlx5_priv *priv = dev->data->dev_private;
16178         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16179         struct mlx5_flow_meter_def_policy *def_policy;
16180         struct mlx5_flow_tbl_resource *jump_tbl;
16181         struct mlx5_flow_tbl_data_entry *tbl_data;
16182         uint8_t egress, transfer;
16183         struct rte_flow_error error;
16184         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16185         int ret;
16186
16187         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16188         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16189         def_policy = mtrmng->def_policy[domain];
16190         if (!def_policy) {
16191                 def_policy = mlx5_malloc(MLX5_MEM_ZERO,
16192                         sizeof(struct mlx5_flow_meter_def_policy),
16193                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
16194                 if (!def_policy) {
16195                         DRV_LOG(ERR, "Failed to alloc "
16196                                         "default policy table.");
16197                         goto def_policy_error;
16198                 }
16199                 mtrmng->def_policy[domain] = def_policy;
16200                 /* Create the meter suffix table with SUFFIX level. */
16201                 jump_tbl = flow_dv_tbl_resource_get(dev,
16202                                 MLX5_FLOW_TABLE_LEVEL_METER,
16203                                 egress, transfer, false, NULL, 0,
16204                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16205                 if (!jump_tbl) {
16206                         DRV_LOG(ERR,
16207                                 "Failed to create meter suffix table.");
16208                         goto def_policy_error;
16209                 }
16210                 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
16211                 tbl_data = container_of(jump_tbl,
16212                                 struct mlx5_flow_tbl_data_entry, tbl);
16213                 def_policy->dr_jump_action[RTE_COLOR_GREEN] =
16214                                                 tbl_data->jump.action;
16215                 acts[RTE_COLOR_GREEN].dv_actions[0] =
16216                                                 tbl_data->jump.action;
16217                 acts[RTE_COLOR_GREEN].actions_n = 1;
16218                 /* Create jump action to the drop table. */
16219                 if (!mtrmng->drop_tbl[domain]) {
16220                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
16221                                 (dev, MLX5_FLOW_TABLE_LEVEL_METER,
16222                                 egress, transfer, false, NULL, 0,
16223                                 0, MLX5_MTR_TABLE_ID_DROP, &error);
16224                         if (!mtrmng->drop_tbl[domain]) {
16225                                 DRV_LOG(ERR, "Failed to create "
16226                                 "meter drop table for default policy.");
16227                                 goto def_policy_error;
16228                         }
16229                 }
16230                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16231                                 struct mlx5_flow_tbl_data_entry, tbl);
16232                 def_policy->dr_jump_action[RTE_COLOR_RED] =
16233                                                 tbl_data->jump.action;
16234                 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
16235                 acts[RTE_COLOR_RED].actions_n = 1;
16236                 /* Create default policy rules. */
16237                 ret = __flow_dv_create_domain_policy_rules(dev,
16238                                         &def_policy->sub_policy,
16239                                         egress, transfer, false, acts);
16240                 if (ret) {
16241                         DRV_LOG(ERR, "Failed to create "
16242                                 "default policy rules.");
16243                                 goto def_policy_error;
16244                 }
16245         }
16246         return 0;
16247 def_policy_error:
16248         __flow_dv_destroy_domain_def_policy(dev,
16249                         (enum mlx5_meter_domain)domain);
16250         return -1;
16251 }
16252
16253 /**
16254  * Create the default policy table set.
16255  *
16256  * @param[in] dev
16257  *   Pointer to Ethernet device.
16258  * @return
16259  *   0 on success, -1 otherwise.
16260  */
16261 static int
16262 flow_dv_create_def_policy(struct rte_eth_dev *dev)
16263 {
16264         struct mlx5_priv *priv = dev->data->dev_private;
16265         int i;
16266
16267         /* Non-termination policy table. */
16268         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16269                 if (!priv->config.dv_esw_en && i == MLX5_MTR_DOMAIN_TRANSFER)
16270                         continue;
16271                 if (__flow_dv_create_domain_def_policy(dev, i)) {
16272                         DRV_LOG(ERR,
16273                         "Failed to create default policy");
16274                         return -1;
16275                 }
16276         }
16277         return 0;
16278 }
16279
16280 /**
16281  * Create the needed meter tables.
16282  * Lock free, (mutex should be acquired by caller).
16283  *
16284  * @param[in] dev
16285  *   Pointer to Ethernet device.
16286  * @param[in] fm
16287  *   Meter information table.
16288  * @param[in] mtr_idx
16289  *   Meter index.
16290  * @param[in] domain_bitmap
16291  *   Domain bitmap.
16292  * @return
16293  *   0 on success, -1 otherwise.
16294  */
16295 static int
16296 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
16297                         struct mlx5_flow_meter_info *fm,
16298                         uint32_t mtr_idx,
16299                         uint8_t domain_bitmap)
16300 {
16301         struct mlx5_priv *priv = dev->data->dev_private;
16302         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16303         struct rte_flow_error error;
16304         struct mlx5_flow_tbl_data_entry *tbl_data;
16305         uint8_t egress, transfer;
16306         void *actions[METER_ACTIONS];
16307         int domain, ret, i;
16308         struct mlx5_flow_counter *cnt;
16309         struct mlx5_flow_dv_match_params value = {
16310                 .size = sizeof(value.buf),
16311         };
16312         struct mlx5_flow_dv_match_params matcher_para = {
16313                 .size = sizeof(matcher_para.buf),
16314         };
16315         int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
16316                                                      0, &error);
16317         uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
16318         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
16319         struct mlx5_list_entry *entry;
16320         struct mlx5_flow_dv_matcher matcher = {
16321                 .mask = {
16322                         .size = sizeof(matcher.mask.buf),
16323                 },
16324         };
16325         struct mlx5_flow_dv_matcher *drop_matcher;
16326         struct mlx5_flow_cb_ctx ctx = {
16327                 .error = &error,
16328                 .data = &matcher,
16329         };
16330         uint8_t misc_mask;
16331
16332         if (!priv->mtr_en || mtr_id_reg_c < 0) {
16333                 rte_errno = ENOTSUP;
16334                 return -1;
16335         }
16336         for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
16337                 if (!(domain_bitmap & (1 << domain)) ||
16338                         (mtrmng->def_rule[domain] && !fm->drop_cnt))
16339                         continue;
16340                 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16341                 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16342                 /* Create the drop table with METER DROP level. */
16343                 if (!mtrmng->drop_tbl[domain]) {
16344                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
16345                                         MLX5_FLOW_TABLE_LEVEL_METER,
16346                                         egress, transfer, false, NULL, 0,
16347                                         0, MLX5_MTR_TABLE_ID_DROP, &error);
16348                         if (!mtrmng->drop_tbl[domain]) {
16349                                 DRV_LOG(ERR, "Failed to create meter drop table.");
16350                                 goto policy_error;
16351                         }
16352                 }
16353                 /* Create default matcher in drop table. */
16354                 matcher.tbl = mtrmng->drop_tbl[domain],
16355                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16356                                 struct mlx5_flow_tbl_data_entry, tbl);
16357                 if (!mtrmng->def_matcher[domain]) {
16358                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16359                                        (enum modify_reg)mtr_id_reg_c,
16360                                        0, 0);
16361                         matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
16362                         matcher.crc = rte_raw_cksum
16363                                         ((const void *)matcher.mask.buf,
16364                                         matcher.mask.size);
16365                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16366                         if (!entry) {
16367                                 DRV_LOG(ERR, "Failed to register meter "
16368                                 "drop default matcher.");
16369                                 goto policy_error;
16370                         }
16371                         mtrmng->def_matcher[domain] = container_of(entry,
16372                         struct mlx5_flow_dv_matcher, entry);
16373                 }
16374                 /* Create default rule in drop table. */
16375                 if (!mtrmng->def_rule[domain]) {
16376                         i = 0;
16377                         actions[i++] = priv->sh->dr_drop_action;
16378                         flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16379                                 (enum modify_reg)mtr_id_reg_c, 0, 0);
16380                         misc_mask = flow_dv_matcher_enable(value.buf);
16381                         __flow_dv_adjust_buf_size(&value.size, misc_mask);
16382                         ret = mlx5_flow_os_create_flow
16383                                 (mtrmng->def_matcher[domain]->matcher_object,
16384                                 (void *)&value, i, actions,
16385                                 &mtrmng->def_rule[domain]);
16386                         if (ret) {
16387                                 DRV_LOG(ERR, "Failed to create meter "
16388                                 "default drop rule for drop table.");
16389                                 goto policy_error;
16390                         }
16391                 }
16392                 if (!fm->drop_cnt)
16393                         continue;
16394                 MLX5_ASSERT(mtrmng->max_mtr_bits);
16395                 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
16396                         /* Create matchers for Drop. */
16397                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16398                                         (enum modify_reg)mtr_id_reg_c, 0,
16399                                         (mtr_id_mask << mtr_id_offset));
16400                         matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
16401                         matcher.crc = rte_raw_cksum
16402                                         ((const void *)matcher.mask.buf,
16403                                         matcher.mask.size);
16404                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16405                         if (!entry) {
16406                                 DRV_LOG(ERR,
16407                                 "Failed to register meter drop matcher.");
16408                                 goto policy_error;
16409                         }
16410                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
16411                                 container_of(entry, struct mlx5_flow_dv_matcher,
16412                                              entry);
16413                 }
16414                 drop_matcher =
16415                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
16416                 /* Create drop rule, matching meter_id only. */
16417                 flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16418                                 (enum modify_reg)mtr_id_reg_c,
16419                                 (mtr_idx << mtr_id_offset), UINT32_MAX);
16420                 i = 0;
16421                 cnt = flow_dv_counter_get_by_idx(dev,
16422                                         fm->drop_cnt, NULL);
16423                 actions[i++] = cnt->action;
16424                 actions[i++] = priv->sh->dr_drop_action;
16425                 misc_mask = flow_dv_matcher_enable(value.buf);
16426                 __flow_dv_adjust_buf_size(&value.size, misc_mask);
16427                 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
16428                                                (void *)&value, i, actions,
16429                                                &fm->drop_rule[domain]);
16430                 if (ret) {
16431                         DRV_LOG(ERR, "Failed to create meter "
16432                                 "drop rule for drop table.");
16433                                 goto policy_error;
16434                 }
16435         }
16436         return 0;
16437 policy_error:
16438         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16439                 if (fm->drop_rule[i]) {
16440                         claim_zero(mlx5_flow_os_destroy_flow
16441                                 (fm->drop_rule[i]));
16442                         fm->drop_rule[i] = NULL;
16443                 }
16444         }
16445         return -1;
16446 }
16447
16448 static struct mlx5_flow_meter_sub_policy *
16449 __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
16450                 struct mlx5_flow_meter_policy *mtr_policy,
16451                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS],
16452                 struct mlx5_flow_meter_sub_policy *next_sub_policy,
16453                 bool *is_reuse)
16454 {
16455         struct mlx5_priv *priv = dev->data->dev_private;
16456         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16457         uint32_t sub_policy_idx = 0;
16458         uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
16459         uint32_t i, j;
16460         struct mlx5_hrxq *hrxq;
16461         struct mlx5_flow_handle dh;
16462         struct mlx5_meter_policy_action_container *act_cnt;
16463         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16464         uint16_t sub_policy_num;
16465
16466         rte_spinlock_lock(&mtr_policy->sl);
16467         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16468                 if (!rss_desc[i])
16469                         continue;
16470                 hrxq_idx[i] = mlx5_hrxq_get(dev, rss_desc[i]);
16471                 if (!hrxq_idx[i]) {
16472                         rte_spinlock_unlock(&mtr_policy->sl);
16473                         return NULL;
16474                 }
16475         }
16476         sub_policy_num = (mtr_policy->sub_policy_num >>
16477                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16478                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16479         for (i = 0; i < sub_policy_num;
16480                 i++) {
16481                 for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) {
16482                         if (rss_desc[j] &&
16483                                 hrxq_idx[j] !=
16484                         mtr_policy->sub_policys[domain][i]->rix_hrxq[j])
16485                                 break;
16486                 }
16487                 if (j >= MLX5_MTR_RTE_COLORS) {
16488                         /*
16489                          * Found the sub policy table with
16490                          * the same queue per color
16491                          */
16492                         rte_spinlock_unlock(&mtr_policy->sl);
16493                         for (j = 0; j < MLX5_MTR_RTE_COLORS; j++)
16494                                 mlx5_hrxq_release(dev, hrxq_idx[j]);
16495                         *is_reuse = true;
16496                         return mtr_policy->sub_policys[domain][i];
16497                 }
16498         }
16499         /* Create sub policy. */
16500         if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
16501                 /* Reuse the first dummy sub_policy*/
16502                 sub_policy = mtr_policy->sub_policys[domain][0];
16503                 sub_policy_idx = sub_policy->idx;
16504         } else {
16505                 sub_policy = mlx5_ipool_zmalloc
16506                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16507                                 &sub_policy_idx);
16508                 if (!sub_policy ||
16509                         sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
16510                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
16511                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
16512                         goto rss_sub_policy_error;
16513                 }
16514                 sub_policy->idx = sub_policy_idx;
16515                 sub_policy->main_policy = mtr_policy;
16516         }
16517         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16518                 if (!rss_desc[i])
16519                         continue;
16520                 sub_policy->rix_hrxq[i] = hrxq_idx[i];
16521                 if (mtr_policy->is_hierarchy) {
16522                         act_cnt = &mtr_policy->act_cnt[i];
16523                         act_cnt->next_sub_policy = next_sub_policy;
16524                         mlx5_hrxq_release(dev, hrxq_idx[i]);
16525                 } else {
16526                         /*
16527                          * Overwrite the last action from
16528                          * RSS action to Queue action.
16529                          */
16530                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
16531                                 hrxq_idx[i]);
16532                         if (!hrxq) {
16533                                 DRV_LOG(ERR, "Failed to create policy hrxq");
16534                                 goto rss_sub_policy_error;
16535                         }
16536                         act_cnt = &mtr_policy->act_cnt[i];
16537                         if (act_cnt->rix_mark || act_cnt->modify_hdr) {
16538                                 memset(&dh, 0, sizeof(struct mlx5_flow_handle));
16539                                 if (act_cnt->rix_mark)
16540                                         dh.mark = 1;
16541                                 dh.fate_action = MLX5_FLOW_FATE_QUEUE;
16542                                 dh.rix_hrxq = hrxq_idx[i];
16543                                 flow_drv_rxq_flags_set(dev, &dh);
16544                         }
16545                 }
16546         }
16547         if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16548                 sub_policy, domain)) {
16549                 DRV_LOG(ERR, "Failed to create policy "
16550                         "rules per domain.");
16551                 goto rss_sub_policy_error;
16552         }
16553         if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16554                 i = (mtr_policy->sub_policy_num >>
16555                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16556                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16557                 mtr_policy->sub_policys[domain][i] = sub_policy;
16558                 i++;
16559                 if (i > MLX5_MTR_RSS_MAX_SUB_POLICY)
16560                         goto rss_sub_policy_error;
16561                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16562                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16563                 mtr_policy->sub_policy_num |=
16564                         (i & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16565                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16566         }
16567         rte_spinlock_unlock(&mtr_policy->sl);
16568         *is_reuse = false;
16569         return sub_policy;
16570 rss_sub_policy_error:
16571         if (sub_policy) {
16572                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16573                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16574                         i = (mtr_policy->sub_policy_num >>
16575                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16576                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16577                         mtr_policy->sub_policys[domain][i] = NULL;
16578                         mlx5_ipool_free
16579                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16580                                         sub_policy->idx);
16581                 }
16582         }
16583         rte_spinlock_unlock(&mtr_policy->sl);
16584         return NULL;
16585 }
16586
16587 /**
16588  * Find the policy table for prefix table with RSS.
16589  *
16590  * @param[in] dev
16591  *   Pointer to Ethernet device.
16592  * @param[in] mtr_policy
16593  *   Pointer to meter policy table.
16594  * @param[in] rss_desc
16595  *   Pointer to rss_desc
16596  * @return
16597  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
16598  */
16599 static struct mlx5_flow_meter_sub_policy *
16600 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
16601                 struct mlx5_flow_meter_policy *mtr_policy,
16602                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
16603 {
16604         struct mlx5_priv *priv = dev->data->dev_private;
16605         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16606         struct mlx5_flow_meter_info *next_fm;
16607         struct mlx5_flow_meter_policy *next_policy;
16608         struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL;
16609         struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM];
16610         struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM];
16611         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16612         bool reuse_sub_policy;
16613         uint32_t i = 0;
16614         uint32_t j = 0;
16615
16616         while (true) {
16617                 /* Iterate hierarchy to get all policies in this hierarchy. */
16618                 policies[i++] = mtr_policy;
16619                 if (!mtr_policy->is_hierarchy)
16620                         break;
16621                 if (i >= MLX5_MTR_CHAIN_MAX_NUM) {
16622                         DRV_LOG(ERR, "Exceed max meter number in hierarchy.");
16623                         return NULL;
16624                 }
16625                 next_fm = mlx5_flow_meter_find(priv,
16626                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16627                 if (!next_fm) {
16628                         DRV_LOG(ERR, "Failed to get next meter in hierarchy.");
16629                         return NULL;
16630                 }
16631                 next_policy =
16632                         mlx5_flow_meter_policy_find(dev, next_fm->policy_id,
16633                                                     NULL);
16634                 MLX5_ASSERT(next_policy);
16635                 mtr_policy = next_policy;
16636         }
16637         while (i) {
16638                 /**
16639                  * From last policy to the first one in hierarchy,
16640                  * create/get the sub policy for each of them.
16641                  */
16642                 sub_policy = __flow_dv_meter_get_rss_sub_policy(dev,
16643                                                         policies[--i],
16644                                                         rss_desc,
16645                                                         next_sub_policy,
16646                                                         &reuse_sub_policy);
16647                 if (!sub_policy) {
16648                         DRV_LOG(ERR, "Failed to get the sub policy.");
16649                         goto err_exit;
16650                 }
16651                 if (!reuse_sub_policy)
16652                         sub_policies[j++] = sub_policy;
16653                 next_sub_policy = sub_policy;
16654         }
16655         return sub_policy;
16656 err_exit:
16657         while (j) {
16658                 uint16_t sub_policy_num;
16659
16660                 sub_policy = sub_policies[--j];
16661                 mtr_policy = sub_policy->main_policy;
16662                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16663                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16664                         sub_policy_num = (mtr_policy->sub_policy_num >>
16665                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16666                                 MLX5_MTR_SUB_POLICY_NUM_MASK;
16667                         mtr_policy->sub_policys[domain][sub_policy_num - 1] =
16668                                                                         NULL;
16669                         sub_policy_num--;
16670                         mtr_policy->sub_policy_num &=
16671                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16672                                   (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
16673                         mtr_policy->sub_policy_num |=
16674                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16675                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
16676                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16677                                         sub_policy->idx);
16678                 }
16679         }
16680         return NULL;
16681 }
16682
16683 /**
16684  * Create the sub policy tag rule for all meters in hierarchy.
16685  *
16686  * @param[in] dev
16687  *   Pointer to Ethernet device.
16688  * @param[in] fm
16689  *   Meter information table.
16690  * @param[in] src_port
16691  *   The src port this extra rule should use.
16692  * @param[in] item
16693  *   The src port match item.
16694  * @param[out] error
16695  *   Perform verbose error reporting if not NULL.
16696  * @return
16697  *   0 on success, a negative errno value otherwise and rte_errno is set.
16698  */
16699 static int
16700 flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
16701                                 struct mlx5_flow_meter_info *fm,
16702                                 int32_t src_port,
16703                                 const struct rte_flow_item *item,
16704                                 struct rte_flow_error *error)
16705 {
16706         struct mlx5_priv *priv = dev->data->dev_private;
16707         struct mlx5_flow_meter_policy *mtr_policy;
16708         struct mlx5_flow_meter_sub_policy *sub_policy;
16709         struct mlx5_flow_meter_info *next_fm = NULL;
16710         struct mlx5_flow_meter_policy *next_policy;
16711         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16712         struct mlx5_flow_tbl_data_entry *tbl_data;
16713         struct mlx5_sub_policy_color_rule *color_rule;
16714         struct mlx5_meter_policy_acts acts;
16715         uint32_t color_reg_c_idx;
16716         bool mtr_first = (src_port != UINT16_MAX) ? true : false;
16717         struct rte_flow_attr attr = {
16718                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16719                 .priority = 0,
16720                 .ingress = 0,
16721                 .egress = 0,
16722                 .transfer = 1,
16723                 .reserved = 0,
16724         };
16725         uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
16726         int i;
16727
16728         mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
16729         MLX5_ASSERT(mtr_policy);
16730         if (!mtr_policy->is_hierarchy)
16731                 return 0;
16732         next_fm = mlx5_flow_meter_find(priv,
16733                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16734         if (!next_fm) {
16735                 return rte_flow_error_set(error, EINVAL,
16736                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
16737                                 "Failed to find next meter in hierarchy.");
16738         }
16739         if (!next_fm->drop_cnt)
16740                 goto exit;
16741         color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error);
16742         sub_policy = mtr_policy->sub_policys[domain][0];
16743         for (i = 0; i < RTE_COLORS; i++) {
16744                 bool rule_exist = false;
16745                 struct mlx5_meter_policy_action_container *act_cnt;
16746
16747                 if (i >= RTE_COLOR_YELLOW)
16748                         break;
16749                 TAILQ_FOREACH(color_rule,
16750                               &sub_policy->color_rules[i], next_port)
16751                         if (color_rule->src_port == src_port) {
16752                                 rule_exist = true;
16753                                 break;
16754                         }
16755                 if (rule_exist)
16756                         continue;
16757                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
16758                                 sizeof(struct mlx5_sub_policy_color_rule),
16759                                 0, SOCKET_ID_ANY);
16760                 if (!color_rule)
16761                         return rte_flow_error_set(error, ENOMEM,
16762                                 RTE_FLOW_ERROR_TYPE_ACTION,
16763                                 NULL, "No memory to create tag color rule.");
16764                 color_rule->src_port = src_port;
16765                 attr.priority = i;
16766                 next_policy = mlx5_flow_meter_policy_find(dev,
16767                                                 next_fm->policy_id, NULL);
16768                 MLX5_ASSERT(next_policy);
16769                 next_sub_policy = next_policy->sub_policys[domain][0];
16770                 tbl_data = container_of(next_sub_policy->tbl_rsc,
16771                                         struct mlx5_flow_tbl_data_entry, tbl);
16772                 act_cnt = &mtr_policy->act_cnt[i];
16773                 if (mtr_first) {
16774                         acts.dv_actions[0] = next_fm->meter_action;
16775                         acts.dv_actions[1] = act_cnt->modify_hdr->action;
16776                 } else {
16777                         acts.dv_actions[0] = act_cnt->modify_hdr->action;
16778                         acts.dv_actions[1] = next_fm->meter_action;
16779                 }
16780                 acts.dv_actions[2] = tbl_data->jump.action;
16781                 acts.actions_n = 3;
16782                 if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) {
16783                         next_fm = NULL;
16784                         goto err_exit;
16785                 }
16786                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
16787                                         i, sub_policy, &attr, true, item,
16788                                         &color_rule->matcher, error)) {
16789                         rte_flow_error_set(error, errno,
16790                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
16791                                 "Failed to create hierarchy meter matcher.");
16792                         goto err_exit;
16793                 }
16794                 if (__flow_dv_create_policy_flow(dev, color_reg_c_idx,
16795                                         (enum rte_color)i,
16796                                         color_rule->matcher->matcher_object,
16797                                         acts.actions_n, acts.dv_actions,
16798                                         true, item,
16799                                         &color_rule->rule, &attr)) {
16800                         rte_flow_error_set(error, errno,
16801                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
16802                                 "Failed to create hierarchy meter rule.");
16803                         goto err_exit;
16804                 }
16805                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
16806                                   color_rule, next_port);
16807         }
16808 exit:
16809         /**
16810          * Recursive call to iterate all meters in hierarchy and
16811          * create needed rules.
16812          */
16813         return flow_dv_meter_hierarchy_rule_create(dev, next_fm,
16814                                                 src_port, item, error);
16815 err_exit:
16816         if (color_rule) {
16817                 if (color_rule->rule)
16818                         mlx5_flow_os_destroy_flow(color_rule->rule);
16819                 if (color_rule->matcher) {
16820                         struct mlx5_flow_tbl_data_entry *tbl =
16821                                 container_of(color_rule->matcher->tbl,
16822                                                 typeof(*tbl), tbl);
16823                         mlx5_list_unregister(tbl->matchers,
16824                                                 &color_rule->matcher->entry);
16825                 }
16826                 mlx5_free(color_rule);
16827         }
16828         if (next_fm)
16829                 mlx5_flow_meter_detach(priv, next_fm);
16830         return -rte_errno;
16831 }
16832
16833 /**
16834  * Destroy the sub policy table with RX queue.
16835  *
16836  * @param[in] dev
16837  *   Pointer to Ethernet device.
16838  * @param[in] mtr_policy
16839  *   Pointer to meter policy table.
16840  */
16841 static void
16842 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
16843                 struct mlx5_flow_meter_policy *mtr_policy)
16844 {
16845         struct mlx5_priv *priv = dev->data->dev_private;
16846         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16847         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16848         uint32_t i, j;
16849         uint16_t sub_policy_num, new_policy_num;
16850
16851         rte_spinlock_lock(&mtr_policy->sl);
16852         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16853                 switch (mtr_policy->act_cnt[i].fate_action) {
16854                 case MLX5_FLOW_FATE_SHARED_RSS:
16855                         sub_policy_num = (mtr_policy->sub_policy_num >>
16856                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16857                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16858                         new_policy_num = sub_policy_num;
16859                         for (j = 0; j < sub_policy_num; j++) {
16860                                 sub_policy =
16861                                         mtr_policy->sub_policys[domain][j];
16862                                 if (sub_policy) {
16863                                         __flow_dv_destroy_sub_policy_rules(dev,
16864                                                 sub_policy);
16865                                 if (sub_policy !=
16866                                         mtr_policy->sub_policys[domain][0]) {
16867                                         mtr_policy->sub_policys[domain][j] =
16868                                                                 NULL;
16869                                         mlx5_ipool_free
16870                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16871                                                 sub_policy->idx);
16872                                                 new_policy_num--;
16873                                         }
16874                                 }
16875                         }
16876                         if (new_policy_num != sub_policy_num) {
16877                                 mtr_policy->sub_policy_num &=
16878                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16879                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16880                                 mtr_policy->sub_policy_num |=
16881                                 (new_policy_num &
16882                                         MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16883                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16884                         }
16885                         break;
16886                 case MLX5_FLOW_FATE_QUEUE:
16887                         sub_policy = mtr_policy->sub_policys[domain][0];
16888                         __flow_dv_destroy_sub_policy_rules(dev,
16889                                                 sub_policy);
16890                         break;
16891                 default:
16892                         /*Other actions without queue and do nothing*/
16893                         break;
16894                 }
16895         }
16896         rte_spinlock_unlock(&mtr_policy->sl);
16897 }
16898
16899 /**
16900  * Validate the batch counter support in root table.
16901  *
16902  * Create a simple flow with invalid counter and drop action on root table to
16903  * validate if batch counter with offset on root table is supported or not.
16904  *
16905  * @param[in] dev
16906  *   Pointer to rte_eth_dev structure.
16907  *
16908  * @return
16909  *   0 on success, a negative errno value otherwise and rte_errno is set.
16910  */
16911 int
16912 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
16913 {
16914         struct mlx5_priv *priv = dev->data->dev_private;
16915         struct mlx5_dev_ctx_shared *sh = priv->sh;
16916         struct mlx5_flow_dv_match_params mask = {
16917                 .size = sizeof(mask.buf),
16918         };
16919         struct mlx5_flow_dv_match_params value = {
16920                 .size = sizeof(value.buf),
16921         };
16922         struct mlx5dv_flow_matcher_attr dv_attr = {
16923                 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS,
16924                 .priority = 0,
16925                 .match_criteria_enable = 0,
16926                 .match_mask = (void *)&mask,
16927         };
16928         void *actions[2] = { 0 };
16929         struct mlx5_flow_tbl_resource *tbl = NULL;
16930         struct mlx5_devx_obj *dcs = NULL;
16931         void *matcher = NULL;
16932         void *flow = NULL;
16933         int ret = -1;
16934
16935         tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL,
16936                                         0, 0, 0, NULL);
16937         if (!tbl)
16938                 goto err;
16939         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
16940         if (!dcs)
16941                 goto err;
16942         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
16943                                                     &actions[0]);
16944         if (ret)
16945                 goto err;
16946         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
16947         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
16948         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
16949                                                &matcher);
16950         if (ret)
16951                 goto err;
16952         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
16953         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
16954                                        actions, &flow);
16955 err:
16956         /*
16957          * If batch counter with offset is not supported, the driver will not
16958          * validate the invalid offset value, flow create should success.
16959          * In this case, it means batch counter is not supported in root table.
16960          *
16961          * Otherwise, if flow create is failed, counter offset is supported.
16962          */
16963         if (flow) {
16964                 DRV_LOG(INFO, "Batch counter is not supported in root "
16965                               "table. Switch to fallback mode.");
16966                 rte_errno = ENOTSUP;
16967                 ret = -rte_errno;
16968                 claim_zero(mlx5_flow_os_destroy_flow(flow));
16969         } else {
16970                 /* Check matcher to make sure validate fail at flow create. */
16971                 if (!matcher || (matcher && errno != EINVAL))
16972                         DRV_LOG(ERR, "Unexpected error in counter offset "
16973                                      "support detection");
16974                 ret = 0;
16975         }
16976         if (actions[0])
16977                 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0]));
16978         if (matcher)
16979                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
16980         if (tbl)
16981                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
16982         if (dcs)
16983                 claim_zero(mlx5_devx_cmd_destroy(dcs));
16984         return ret;
16985 }
16986
16987 /**
16988  * Query a devx counter.
16989  *
16990  * @param[in] dev
16991  *   Pointer to the Ethernet device structure.
16992  * @param[in] cnt
16993  *   Index to the flow counter.
16994  * @param[in] clear
16995  *   Set to clear the counter statistics.
16996  * @param[out] pkts
16997  *   The statistics value of packets.
16998  * @param[out] bytes
16999  *   The statistics value of bytes.
17000  *
17001  * @return
17002  *   0 on success, otherwise return -1.
17003  */
17004 static int
17005 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
17006                       uint64_t *pkts, uint64_t *bytes)
17007 {
17008         struct mlx5_priv *priv = dev->data->dev_private;
17009         struct mlx5_flow_counter *cnt;
17010         uint64_t inn_pkts, inn_bytes;
17011         int ret;
17012
17013         if (!priv->config.devx)
17014                 return -1;
17015
17016         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
17017         if (ret)
17018                 return -1;
17019         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
17020         *pkts = inn_pkts - cnt->hits;
17021         *bytes = inn_bytes - cnt->bytes;
17022         if (clear) {
17023                 cnt->hits = inn_pkts;
17024                 cnt->bytes = inn_bytes;
17025         }
17026         return 0;
17027 }
17028
17029 /**
17030  * Get aged-out flows.
17031  *
17032  * @param[in] dev
17033  *   Pointer to the Ethernet device structure.
17034  * @param[in] context
17035  *   The address of an array of pointers to the aged-out flows contexts.
17036  * @param[in] nb_contexts
17037  *   The length of context array pointers.
17038  * @param[out] error
17039  *   Perform verbose error reporting if not NULL. Initialized in case of
17040  *   error only.
17041  *
17042  * @return
17043  *   how many contexts get in success, otherwise negative errno value.
17044  *   if nb_contexts is 0, return the amount of all aged contexts.
17045  *   if nb_contexts is not 0 , return the amount of aged flows reported
17046  *   in the context array.
17047  * @note: only stub for now
17048  */
17049 static int
17050 flow_get_aged_flows(struct rte_eth_dev *dev,
17051                     void **context,
17052                     uint32_t nb_contexts,
17053                     struct rte_flow_error *error)
17054 {
17055         struct mlx5_priv *priv = dev->data->dev_private;
17056         struct mlx5_age_info *age_info;
17057         struct mlx5_age_param *age_param;
17058         struct mlx5_flow_counter *counter;
17059         struct mlx5_aso_age_action *act;
17060         int nb_flows = 0;
17061
17062         if (nb_contexts && !context)
17063                 return rte_flow_error_set(error, EINVAL,
17064                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17065                                           NULL, "empty context");
17066         age_info = GET_PORT_AGE_INFO(priv);
17067         rte_spinlock_lock(&age_info->aged_sl);
17068         LIST_FOREACH(act, &age_info->aged_aso, next) {
17069                 nb_flows++;
17070                 if (nb_contexts) {
17071                         context[nb_flows - 1] =
17072                                                 act->age_params.context;
17073                         if (!(--nb_contexts))
17074                                 break;
17075                 }
17076         }
17077         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
17078                 nb_flows++;
17079                 if (nb_contexts) {
17080                         age_param = MLX5_CNT_TO_AGE(counter);
17081                         context[nb_flows - 1] = age_param->context;
17082                         if (!(--nb_contexts))
17083                                 break;
17084                 }
17085         }
17086         rte_spinlock_unlock(&age_info->aged_sl);
17087         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
17088         return nb_flows;
17089 }
17090
17091 /*
17092  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
17093  */
17094 static uint32_t
17095 flow_dv_counter_allocate(struct rte_eth_dev *dev)
17096 {
17097         return flow_dv_counter_alloc(dev, 0);
17098 }
17099
17100 /**
17101  * Validate indirect action.
17102  * Dispatcher for action type specific validation.
17103  *
17104  * @param[in] dev
17105  *   Pointer to the Ethernet device structure.
17106  * @param[in] conf
17107  *   Indirect action configuration.
17108  * @param[in] action
17109  *   The indirect action object to validate.
17110  * @param[out] error
17111  *   Perform verbose error reporting if not NULL. Initialized in case of
17112  *   error only.
17113  *
17114  * @return
17115  *   0 on success, otherwise negative errno value.
17116  */
17117 static int
17118 flow_dv_action_validate(struct rte_eth_dev *dev,
17119                         const struct rte_flow_indir_action_conf *conf,
17120                         const struct rte_flow_action *action,
17121                         struct rte_flow_error *err)
17122 {
17123         struct mlx5_priv *priv = dev->data->dev_private;
17124
17125         RTE_SET_USED(conf);
17126         switch (action->type) {
17127         case RTE_FLOW_ACTION_TYPE_RSS:
17128                 /*
17129                  * priv->obj_ops is set according to driver capabilities.
17130                  * When DevX capabilities are
17131                  * sufficient, it is set to devx_obj_ops.
17132                  * Otherwise, it is set to ibv_obj_ops.
17133                  * ibv_obj_ops doesn't support ind_table_modify operation.
17134                  * In this case the indirect RSS action can't be used.
17135                  */
17136                 if (priv->obj_ops.ind_table_modify == NULL)
17137                         return rte_flow_error_set
17138                                         (err, ENOTSUP,
17139                                          RTE_FLOW_ERROR_TYPE_ACTION,
17140                                          NULL,
17141                                          "Indirect RSS action not supported");
17142                 return mlx5_validate_action_rss(dev, action, err);
17143         case RTE_FLOW_ACTION_TYPE_AGE:
17144                 if (!priv->sh->aso_age_mng)
17145                         return rte_flow_error_set(err, ENOTSUP,
17146                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17147                                                 NULL,
17148                                                 "Indirect age action not supported");
17149                 return flow_dv_validate_action_age(0, action, dev, err);
17150         case RTE_FLOW_ACTION_TYPE_COUNT:
17151                 /*
17152                  * There are two mechanisms to share the action count.
17153                  * The old mechanism uses the shared field to share, while the
17154                  * new mechanism uses the indirect action API.
17155                  * This validation comes to make sure that the two mechanisms
17156                  * are not combined.
17157                  */
17158                 if (is_shared_action_count(action))
17159                         return rte_flow_error_set(err, ENOTSUP,
17160                                                   RTE_FLOW_ERROR_TYPE_ACTION,
17161                                                   NULL,
17162                                                   "Mix shared and indirect counter is not supported");
17163                 return flow_dv_validate_action_count(dev, true, 0, err);
17164         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
17165                 if (!priv->sh->ct_aso_en)
17166                         return rte_flow_error_set(err, ENOTSUP,
17167                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17168                                         "ASO CT is not supported");
17169                 return mlx5_validate_action_ct(dev, action->conf, err);
17170         default:
17171                 return rte_flow_error_set(err, ENOTSUP,
17172                                           RTE_FLOW_ERROR_TYPE_ACTION,
17173                                           NULL,
17174                                           "action type not supported");
17175         }
17176 }
17177
17178 /**
17179  * Validate the meter hierarchy chain for meter policy.
17180  *
17181  * @param[in] dev
17182  *   Pointer to the Ethernet device structure.
17183  * @param[in] meter_id
17184  *   Meter id.
17185  * @param[in] action_flags
17186  *   Holds the actions detected until now.
17187  * @param[out] is_rss
17188  *   Is RSS or not.
17189  * @param[out] hierarchy_domain
17190  *   The domain bitmap for hierarchy policy.
17191  * @param[out] error
17192  *   Perform verbose error reporting if not NULL. Initialized in case of
17193  *   error only.
17194  *
17195  * @return
17196  *   0 on success, otherwise negative errno value with error set.
17197  */
17198 static int
17199 flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
17200                                   uint32_t meter_id,
17201                                   uint64_t action_flags,
17202                                   bool *is_rss,
17203                                   uint8_t *hierarchy_domain,
17204                                   struct rte_mtr_error *error)
17205 {
17206         struct mlx5_priv *priv = dev->data->dev_private;
17207         struct mlx5_flow_meter_info *fm;
17208         struct mlx5_flow_meter_policy *policy;
17209         uint8_t cnt = 1;
17210
17211         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
17212                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17213                 return -rte_mtr_error_set(error, EINVAL,
17214                                         RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN,
17215                                         NULL,
17216                                         "Multiple fate actions not supported.");
17217         while (true) {
17218                 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
17219                 if (!fm)
17220                         return -rte_mtr_error_set(error, EINVAL,
17221                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17222                                         "Meter not found in meter hierarchy.");
17223                 if (fm->def_policy)
17224                         return -rte_mtr_error_set(error, EINVAL,
17225                                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17226                         "Non termination meter not supported in hierarchy.");
17227                 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
17228                 MLX5_ASSERT(policy);
17229                 if (!policy->is_hierarchy) {
17230                         if (policy->transfer)
17231                                 *hierarchy_domain |=
17232                                                 MLX5_MTR_DOMAIN_TRANSFER_BIT;
17233                         if (policy->ingress)
17234                                 *hierarchy_domain |=
17235                                                 MLX5_MTR_DOMAIN_INGRESS_BIT;
17236                         if (policy->egress)
17237                                 *hierarchy_domain |= MLX5_MTR_DOMAIN_EGRESS_BIT;
17238                         *is_rss = policy->is_rss;
17239                         break;
17240                 }
17241                 meter_id = policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id;
17242                 if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM)
17243                         return -rte_mtr_error_set(error, EINVAL,
17244                                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
17245                                         "Exceed max hierarchy meter number.");
17246         }
17247         return 0;
17248 }
17249
17250 /**
17251  * Validate meter policy actions.
17252  * Dispatcher for action type specific validation.
17253  *
17254  * @param[in] dev
17255  *   Pointer to the Ethernet device structure.
17256  * @param[in] action
17257  *   The meter policy action object to validate.
17258  * @param[in] attr
17259  *   Attributes of flow to determine steering domain.
17260  * @param[out] error
17261  *   Perform verbose error reporting if not NULL. Initialized in case of
17262  *   error only.
17263  *
17264  * @return
17265  *   0 on success, otherwise negative errno value.
17266  */
17267 static int
17268 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
17269                         const struct rte_flow_action *actions[RTE_COLORS],
17270                         struct rte_flow_attr *attr,
17271                         bool *is_rss,
17272                         uint8_t *domain_bitmap,
17273                         bool *is_def_policy,
17274                         struct rte_mtr_error *error)
17275 {
17276         struct mlx5_priv *priv = dev->data->dev_private;
17277         struct mlx5_dev_config *dev_conf = &priv->config;
17278         const struct rte_flow_action *act;
17279         uint64_t action_flags = 0;
17280         int actions_n;
17281         int i, ret;
17282         struct rte_flow_error flow_err;
17283         uint8_t domain_color[RTE_COLORS] = {0};
17284         uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
17285         uint8_t hierarchy_domain = 0;
17286         const struct rte_flow_action_meter *mtr;
17287
17288         if (!priv->config.dv_esw_en)
17289                 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
17290         *domain_bitmap = def_domain;
17291         if (actions[RTE_COLOR_YELLOW] &&
17292                 actions[RTE_COLOR_YELLOW]->type != RTE_FLOW_ACTION_TYPE_END)
17293                 return -rte_mtr_error_set(error, ENOTSUP,
17294                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17295                                 NULL,
17296                                 "Yellow color does not support any action.");
17297         if (actions[RTE_COLOR_YELLOW] &&
17298                 actions[RTE_COLOR_YELLOW]->type != RTE_FLOW_ACTION_TYPE_DROP)
17299                 return -rte_mtr_error_set(error, ENOTSUP,
17300                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17301                                 NULL, "Red color only supports drop action.");
17302         /*
17303          * Check default policy actions:
17304          * Green/Yellow: no action, Red: drop action
17305          */
17306         if ((!actions[RTE_COLOR_GREEN] ||
17307                 actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END)) {
17308                 *is_def_policy = true;
17309                 return 0;
17310         }
17311         flow_err.message = NULL;
17312         for (i = 0; i < RTE_COLORS; i++) {
17313                 act = actions[i];
17314                 for (action_flags = 0, actions_n = 0;
17315                         act && act->type != RTE_FLOW_ACTION_TYPE_END;
17316                         act++) {
17317                         if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
17318                                 return -rte_mtr_error_set(error, ENOTSUP,
17319                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17320                                           NULL, "too many actions");
17321                         switch (act->type) {
17322                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
17323                                 if (!priv->config.dv_esw_en)
17324                                         return -rte_mtr_error_set(error,
17325                                         ENOTSUP,
17326                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17327                                         NULL, "PORT action validate check"
17328                                         " fail for ESW disable");
17329                                 ret = flow_dv_validate_action_port_id(dev,
17330                                                 action_flags,
17331                                                 act, attr, &flow_err);
17332                                 if (ret)
17333                                         return -rte_mtr_error_set(error,
17334                                         ENOTSUP,
17335                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17336                                         NULL, flow_err.message ?
17337                                         flow_err.message :
17338                                         "PORT action validate check fail");
17339                                 ++actions_n;
17340                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
17341                                 break;
17342                         case RTE_FLOW_ACTION_TYPE_MARK:
17343                                 ret = flow_dv_validate_action_mark(dev, act,
17344                                                            action_flags,
17345                                                            attr, &flow_err);
17346                                 if (ret < 0)
17347                                         return -rte_mtr_error_set(error,
17348                                         ENOTSUP,
17349                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17350                                         NULL, flow_err.message ?
17351                                         flow_err.message :
17352                                         "Mark action validate check fail");
17353                                 if (dev_conf->dv_xmeta_en !=
17354                                         MLX5_XMETA_MODE_LEGACY)
17355                                         return -rte_mtr_error_set(error,
17356                                         ENOTSUP,
17357                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17358                                         NULL, "Extend MARK action is "
17359                                         "not supported. Please try use "
17360                                         "default policy for meter.");
17361                                 action_flags |= MLX5_FLOW_ACTION_MARK;
17362                                 ++actions_n;
17363                                 break;
17364                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
17365                                 ret = flow_dv_validate_action_set_tag(dev,
17366                                                         act, action_flags,
17367                                                         attr, &flow_err);
17368                                 if (ret)
17369                                         return -rte_mtr_error_set(error,
17370                                         ENOTSUP,
17371                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17372                                         NULL, flow_err.message ?
17373                                         flow_err.message :
17374                                         "Set tag action validate check fail");
17375                                 /*
17376                                  * Count all modify-header actions
17377                                  * as one action.
17378                                  */
17379                                 if (!(action_flags &
17380                                         MLX5_FLOW_MODIFY_HDR_ACTIONS))
17381                                         ++actions_n;
17382                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
17383                                 break;
17384                         case RTE_FLOW_ACTION_TYPE_DROP:
17385                                 ret = mlx5_flow_validate_action_drop
17386                                         (action_flags,
17387                                         attr, &flow_err);
17388                                 if (ret < 0)
17389                                         return -rte_mtr_error_set(error,
17390                                         ENOTSUP,
17391                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17392                                         NULL, flow_err.message ?
17393                                         flow_err.message :
17394                                         "Drop action validate check fail");
17395                                 action_flags |= MLX5_FLOW_ACTION_DROP;
17396                                 ++actions_n;
17397                                 break;
17398                         case RTE_FLOW_ACTION_TYPE_QUEUE:
17399                                 /*
17400                                  * Check whether extensive
17401                                  * metadata feature is engaged.
17402                                  */
17403                                 if (dev_conf->dv_flow_en &&
17404                                         (dev_conf->dv_xmeta_en !=
17405                                         MLX5_XMETA_MODE_LEGACY) &&
17406                                         mlx5_flow_ext_mreg_supported(dev))
17407                                         return -rte_mtr_error_set(error,
17408                                           ENOTSUP,
17409                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17410                                           NULL, "Queue action with meta "
17411                                           "is not supported. Please try use "
17412                                           "default policy for meter.");
17413                                 ret = mlx5_flow_validate_action_queue(act,
17414                                                         action_flags, dev,
17415                                                         attr, &flow_err);
17416                                 if (ret < 0)
17417                                         return -rte_mtr_error_set(error,
17418                                           ENOTSUP,
17419                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17420                                           NULL, flow_err.message ?
17421                                           flow_err.message :
17422                                           "Queue action validate check fail");
17423                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
17424                                 ++actions_n;
17425                                 break;
17426                         case RTE_FLOW_ACTION_TYPE_RSS:
17427                                 if (dev_conf->dv_flow_en &&
17428                                         (dev_conf->dv_xmeta_en !=
17429                                         MLX5_XMETA_MODE_LEGACY) &&
17430                                         mlx5_flow_ext_mreg_supported(dev))
17431                                         return -rte_mtr_error_set(error,
17432                                           ENOTSUP,
17433                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17434                                           NULL, "RSS action with meta "
17435                                           "is not supported. Please try use "
17436                                           "default policy for meter.");
17437                                 ret = mlx5_validate_action_rss(dev, act,
17438                                                 &flow_err);
17439                                 if (ret < 0)
17440                                         return -rte_mtr_error_set(error,
17441                                           ENOTSUP,
17442                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17443                                           NULL, flow_err.message ?
17444                                           flow_err.message :
17445                                           "RSS action validate check fail");
17446                                 action_flags |= MLX5_FLOW_ACTION_RSS;
17447                                 ++actions_n;
17448                                 *is_rss = true;
17449                                 break;
17450                         case RTE_FLOW_ACTION_TYPE_JUMP:
17451                                 ret = flow_dv_validate_action_jump(dev,
17452                                         NULL, act, action_flags,
17453                                         attr, true, &flow_err);
17454                                 if (ret)
17455                                         return -rte_mtr_error_set(error,
17456                                           ENOTSUP,
17457                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17458                                           NULL, flow_err.message ?
17459                                           flow_err.message :
17460                                           "Jump action validate check fail");
17461                                 ++actions_n;
17462                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
17463                                 break;
17464                         case RTE_FLOW_ACTION_TYPE_METER:
17465                                 if (i != RTE_COLOR_GREEN)
17466                                         return -rte_mtr_error_set(error,
17467                                                 ENOTSUP,
17468                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17469                                                 NULL, flow_err.message ?
17470                                                 flow_err.message :
17471                                   "Meter hierarchy only supports GREEN color.");
17472                                 mtr = act->conf;
17473                                 ret = flow_dv_validate_policy_mtr_hierarchy(dev,
17474                                                         mtr->mtr_id,
17475                                                         action_flags,
17476                                                         is_rss,
17477                                                         &hierarchy_domain,
17478                                                         error);
17479                                 if (ret)
17480                                         return ret;
17481                                 ++actions_n;
17482                                 action_flags |=
17483                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
17484                                 break;
17485                         default:
17486                                 return -rte_mtr_error_set(error, ENOTSUP,
17487                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17488                                         NULL,
17489                                         "Doesn't support optional action");
17490                         }
17491                 }
17492                 /* Yellow is not supported, just skip. */
17493                 if (i == RTE_COLOR_YELLOW)
17494                         continue;
17495                 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
17496                         domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT;
17497                 else if ((action_flags &
17498                         (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) ||
17499                         (action_flags & MLX5_FLOW_ACTION_MARK))
17500                         /*
17501                          * Only support MLX5_XMETA_MODE_LEGACY
17502                          * so MARK action only in ingress domain.
17503                          */
17504                         domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT;
17505                 else if (action_flags &
17506                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
17507                         domain_color[i] = hierarchy_domain;
17508                 else
17509                         domain_color[i] = def_domain;
17510                 /*
17511                  * Validate the drop action mutual exclusion
17512                  * with other actions. Drop action is mutually-exclusive
17513                  * with any other action, except for Count action.
17514                  */
17515                 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
17516                         (action_flags & ~MLX5_FLOW_ACTION_DROP)) {
17517                         return -rte_mtr_error_set(error, ENOTSUP,
17518                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17519                                 NULL, "Drop action is mutually-exclusive "
17520                                 "with any other action");
17521                 }
17522                 /* Eswitch has few restrictions on using items and actions */
17523                 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) {
17524                         if (!mlx5_flow_ext_mreg_supported(dev) &&
17525                                 action_flags & MLX5_FLOW_ACTION_MARK)
17526                                 return -rte_mtr_error_set(error, ENOTSUP,
17527                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17528                                         NULL, "unsupported action MARK");
17529                         if (action_flags & MLX5_FLOW_ACTION_QUEUE)
17530                                 return -rte_mtr_error_set(error, ENOTSUP,
17531                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17532                                         NULL, "unsupported action QUEUE");
17533                         if (action_flags & MLX5_FLOW_ACTION_RSS)
17534                                 return -rte_mtr_error_set(error, ENOTSUP,
17535                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17536                                         NULL, "unsupported action RSS");
17537                         if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17538                                 return -rte_mtr_error_set(error, ENOTSUP,
17539                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17540                                         NULL, "no fate action is found");
17541                 } else {
17542                         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) &&
17543                                 (domain_color[i] &
17544                                 MLX5_MTR_DOMAIN_INGRESS_BIT)) {
17545                                 if ((domain_color[i] &
17546                                         MLX5_MTR_DOMAIN_EGRESS_BIT))
17547                                         domain_color[i] =
17548                                         MLX5_MTR_DOMAIN_EGRESS_BIT;
17549                                 else
17550                                         return -rte_mtr_error_set(error,
17551                                         ENOTSUP,
17552                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17553                                         NULL, "no fate action is found");
17554                         }
17555                 }
17556                 if (domain_color[i] != def_domain)
17557                         *domain_bitmap = domain_color[i];
17558         }
17559         return 0;
17560 }
17561
17562 static int
17563 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
17564 {
17565         struct mlx5_priv *priv = dev->data->dev_private;
17566         int ret = 0;
17567
17568         if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
17569                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain,
17570                                                 flags);
17571                 if (ret != 0)
17572                         return ret;
17573         }
17574         if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
17575                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags);
17576                 if (ret != 0)
17577                         return ret;
17578         }
17579         if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
17580                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags);
17581                 if (ret != 0)
17582                         return ret;
17583         }
17584         return 0;
17585 }
17586
17587 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
17588         .validate = flow_dv_validate,
17589         .prepare = flow_dv_prepare,
17590         .translate = flow_dv_translate,
17591         .apply = flow_dv_apply,
17592         .remove = flow_dv_remove,
17593         .destroy = flow_dv_destroy,
17594         .query = flow_dv_query,
17595         .create_mtr_tbls = flow_dv_create_mtr_tbls,
17596         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls,
17597         .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls,
17598         .create_meter = flow_dv_mtr_alloc,
17599         .free_meter = flow_dv_aso_mtr_release_to_pool,
17600         .validate_mtr_acts = flow_dv_validate_mtr_policy_acts,
17601         .create_mtr_acts = flow_dv_create_mtr_policy_acts,
17602         .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts,
17603         .create_policy_rules = flow_dv_create_policy_rules,
17604         .destroy_policy_rules = flow_dv_destroy_policy_rules,
17605         .create_def_policy = flow_dv_create_def_policy,
17606         .destroy_def_policy = flow_dv_destroy_def_policy,
17607         .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare,
17608         .meter_hierarchy_rule_create = flow_dv_meter_hierarchy_rule_create,
17609         .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq,
17610         .counter_alloc = flow_dv_counter_allocate,
17611         .counter_free = flow_dv_counter_free,
17612         .counter_query = flow_dv_counter_query,
17613         .get_aged_flows = flow_get_aged_flows,
17614         .action_validate = flow_dv_action_validate,
17615         .action_create = flow_dv_action_create,
17616         .action_destroy = flow_dv_action_destroy,
17617         .action_update = flow_dv_action_update,
17618         .action_query = flow_dv_action_query,
17619         .sync_domain = flow_dv_sync_domain,
17620 };
17621
17622 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
17623