drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24 #include <rte_mtr.h>
  25 #include <rte_mtr_driver.h>
  26 #include <rte_tailq.h>
  27
  28 #include <mlx5_glue.h>
  29 #include <mlx5_devx_cmds.h>
  30 #include <mlx5_prm.h>
  31 #include <mlx5_malloc.h>
  32
  33 #include "mlx5_defs.h"
  34 #include "mlx5.h"
  35 #include "mlx5_common_os.h"
  36 #include "mlx5_flow.h"
  37 #include "mlx5_flow_os.h"
  38 #include "mlx5_rx.h"
  39 #include "mlx5_tx.h"
  40 #include "rte_pmd_mlx5.h"
  41
  42 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  43
  44 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  45 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  46 #endif
  47
  48 #ifndef HAVE_MLX5DV_DR_ESWITCH
  49 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  50 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  51 #endif
  52 #endif
  53
  54 #ifndef HAVE_MLX5DV_DR
  55 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  56 #endif
  57
  58 /* VLAN header definitions */
  59 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  60 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  61 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  62 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  63 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  64
  65 union flow_dv_attr {
  66         struct {
  67                 uint32_t valid:1;
  68                 uint32_t ipv4:1;
  69                 uint32_t ipv6:1;
  70                 uint32_t tcp:1;
  71                 uint32_t udp:1;
  72                 uint32_t reserved:27;
  73         };
  74         uint32_t attr;
  75 };
  76
  77 static int
  78 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
  79                              struct mlx5_flow_tbl_resource *tbl);
  80
  81 static int
  82 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  83                                      uint32_t encap_decap_idx);
  84
  85 static int
  86 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  87                                         uint32_t port_id);
  88 static void
  89 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);
  90
  91 static int
  92 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
  93                                   uint32_t rix_jump);
  94
  95 /**
  96  * Initialize flow attributes structure according to flow items' types.
  97  *
  98  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  99  * mode. For tunnel mode, the items to be modified are the outermost ones.
 100  *
 101  * @param[in] item
 102  *   Pointer to item specification.
 103  * @param[out] attr
 104  *   Pointer to flow attributes structure.
 105  * @param[in] dev_flow
 106  *   Pointer to the sub flow.
 107  * @param[in] tunnel_decap
 108  *   Whether action is after tunnel decapsulation.
 109  */
 110 static void
 111 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 112                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 113 {
 114         uint64_t layers = dev_flow->handle->layers;
 115
 116         /*
 117          * If layers is already initialized, it means this dev_flow is the
 118          * suffix flow, the layers flags is set by the prefix flow. Need to
 119          * use the layer flags from prefix flow as the suffix flow may not
 120          * have the user defined items as the flow is split.
 121          */
 122         if (layers) {
 123                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 124                         attr->ipv4 = 1;
 125                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 126                         attr->ipv6 = 1;
 127                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 128                         attr->tcp = 1;
 129                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 130                         attr->udp = 1;
 131                 attr->valid = 1;
 132                 return;
 133         }
 134         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 135                 uint8_t next_protocol = 0xff;
 136                 switch (item->type) {
 137                 case RTE_FLOW_ITEM_TYPE_GRE:
 138                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 139                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 140                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 141                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 142                 case RTE_FLOW_ITEM_TYPE_MPLS:
 143                         if (tunnel_decap)
 144                                 attr->attr = 0;
 145                         break;
 146                 case RTE_FLOW_ITEM_TYPE_IPV4:
 147                         if (!attr->ipv6)
 148                                 attr->ipv4 = 1;
 149                         if (item->mask != NULL &&
 150                             ((const struct rte_flow_item_ipv4 *)
 151                             item->mask)->hdr.next_proto_id)
 152                                 next_protocol =
 153                                     ((const struct rte_flow_item_ipv4 *)
 154                                       (item->spec))->hdr.next_proto_id &
 155                                     ((const struct rte_flow_item_ipv4 *)
 156                                       (item->mask))->hdr.next_proto_id;
 157                         if ((next_protocol == IPPROTO_IPIP ||
 158                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 159                                 attr->attr = 0;
 160                         break;
 161                 case RTE_FLOW_ITEM_TYPE_IPV6:
 162                         if (!attr->ipv4)
 163                                 attr->ipv6 = 1;
 164                         if (item->mask != NULL &&
 165                             ((const struct rte_flow_item_ipv6 *)
 166                             item->mask)->hdr.proto)
 167                                 next_protocol =
 168                                     ((const struct rte_flow_item_ipv6 *)
 169                                       (item->spec))->hdr.proto &
 170                                     ((const struct rte_flow_item_ipv6 *)
 171                                       (item->mask))->hdr.proto;
 172                         if ((next_protocol == IPPROTO_IPIP ||
 173                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 174                                 attr->attr = 0;
 175                         break;
 176                 case RTE_FLOW_ITEM_TYPE_UDP:
 177                         if (!attr->tcp)
 178                                 attr->udp = 1;
 179                         break;
 180                 case RTE_FLOW_ITEM_TYPE_TCP:
 181                         if (!attr->udp)
 182                                 attr->tcp = 1;
 183                         break;
 184                 default:
 185                         break;
 186                 }
 187         }
 188         attr->valid = 1;
 189 }
 190
 191 /**
 192  * Convert rte_mtr_color to mlx5 color.
 193  *
 194  * @param[in] rcol
 195  *   rte_mtr_color.
 196  *
 197  * @return
 198  *   mlx5 color.
 199  */
 200 static int
 201 rte_col_2_mlx5_col(enum rte_color rcol)
 202 {
 203         switch (rcol) {
 204         case RTE_COLOR_GREEN:
 205                 return MLX5_FLOW_COLOR_GREEN;
 206         case RTE_COLOR_YELLOW:
 207                 return MLX5_FLOW_COLOR_YELLOW;
 208         case RTE_COLOR_RED:
 209                 return MLX5_FLOW_COLOR_RED;
 210         default:
 211                 break;
 212         }
 213         return MLX5_FLOW_COLOR_UNDEFINED;
 214 }
 215
 216 struct field_modify_info {
 217         uint32_t size; /* Size of field in protocol header, in bytes. */
 218         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 219         enum mlx5_modification_field id;
 220 };
 221
 222 struct field_modify_info modify_eth[] = {
 223         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 224         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 225         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 226         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 227         {0, 0, 0},
 228 };
 229
 230 struct field_modify_info modify_vlan_out_first_vid[] = {
 231         /* Size in bits !!! */
 232         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv4[] = {
 237         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 239         {4, 12, MLX5_MODI_OUT_SIPV4},
 240         {4, 16, MLX5_MODI_OUT_DIPV4},
 241         {0, 0, 0},
 242 };
 243
 244 struct field_modify_info modify_ipv6[] = {
 245         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 246         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 247         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 248         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 249         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 250         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 251         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 252         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 253         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 254         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 255         {0, 0, 0},
 256 };
 257
 258 struct field_modify_info modify_udp[] = {
 259         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 260         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 261         {0, 0, 0},
 262 };
 263
 264 struct field_modify_info modify_tcp[] = {
 265         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 266         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 267         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 268         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 269         {0, 0, 0},
 270 };
 271
 272 static const struct rte_flow_item *
 273 mlx5_flow_find_tunnel_item(const struct rte_flow_item *item)
 274 {
 275         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 276                 switch (item->type) {
 277                 default:
 278                         break;
 279                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 280                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 281                 case RTE_FLOW_ITEM_TYPE_GRE:
 282                 case RTE_FLOW_ITEM_TYPE_MPLS:
 283                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 284                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 285                         return item;
 286                 case RTE_FLOW_ITEM_TYPE_IPV4:
 287                 case RTE_FLOW_ITEM_TYPE_IPV6:
 288                         if (item[1].type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 289                             item[1].type == RTE_FLOW_ITEM_TYPE_IPV6)
 290                                 return item;
 291                         break;
 292                 }
 293         }
 294         return NULL;
 295 }
 296
 297 static void
 298 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 299                           uint8_t next_protocol, uint64_t *item_flags,
 300                           int *tunnel)
 301 {
 302         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 303                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 304         if (next_protocol == IPPROTO_IPIP) {
 305                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 306                 *tunnel = 1;
 307         }
 308         if (next_protocol == IPPROTO_IPV6) {
 309                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 310                 *tunnel = 1;
 311         }
 312 }
 313
 314 /* Update VLAN's VID/PCP based on input rte_flow_action.
 315  *
 316  * @param[in] action
 317  *   Pointer to struct rte_flow_action.
 318  * @param[out] vlan
 319  *   Pointer to struct rte_vlan_hdr.
 320  */
 321 static void
 322 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 323                          struct rte_vlan_hdr *vlan)
 324 {
 325         uint16_t vlan_tci;
 326         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 327                 vlan_tci =
 328                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 329                                                action->conf)->vlan_pcp;
 330                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 331                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 332                 vlan->vlan_tci |= vlan_tci;
 333         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 334                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 335                 vlan->vlan_tci |= rte_be_to_cpu_16
 336                     (((const struct rte_flow_action_of_set_vlan_vid *)
 337                                              action->conf)->vlan_vid);
 338         }
 339 }
 340
 341 /**
 342  * Fetch 1, 2, 3 or 4 byte field from the byte array
 343  * and return as unsigned integer in host-endian format.
 344  *
 345  * @param[in] data
 346  *   Pointer to data array.
 347  * @param[in] size
 348  *   Size of field to extract.
 349  *
 350  * @return
 351  *   converted field in host endian format.
 352  */
 353 static inline uint32_t
 354 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 355 {
 356         uint32_t ret;
 357
 358         switch (size) {
 359         case 1:
 360                 ret = *data;
 361                 break;
 362         case 2:
 363                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 364                 break;
 365         case 3:
 366                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 367                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 368                 break;
 369         case 4:
 370                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 371                 break;
 372         default:
 373                 MLX5_ASSERT(false);
 374                 ret = 0;
 375                 break;
 376         }
 377         return ret;
 378 }
 379
 380 /**
 381  * Convert modify-header action to DV specification.
 382  *
 383  * Data length of each action is determined by provided field description
 384  * and the item mask. Data bit offset and width of each action is determined
 385  * by provided item mask.
 386  *
 387  * @param[in] item
 388  *   Pointer to item specification.
 389  * @param[in] field
 390  *   Pointer to field modification information.
 391  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 392  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 393  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 394  * @param[in] dcopy
 395  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 396  *   Negative offset value sets the same offset as source offset.
 397  *   size field is ignored, value is taken from source field.
 398  * @param[in,out] resource
 399  *   Pointer to the modify-header resource.
 400  * @param[in] type
 401  *   Type of modification.
 402  * @param[out] error
 403  *   Pointer to the error structure.
 404  *
 405  * @return
 406  *   0 on success, a negative errno value otherwise and rte_errno is set.
 407  */
 408 static int
 409 flow_dv_convert_modify_action(struct rte_flow_item *item,
 410                               struct field_modify_info *field,
 411                               struct field_modify_info *dcopy,
 412                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 413                               uint32_t type, struct rte_flow_error *error)
 414 {
 415         uint32_t i = resource->actions_num;
 416         struct mlx5_modification_cmd *actions = resource->actions;
 417         uint32_t carry_b = 0;
 418
 419         /*
 420          * The item and mask are provided in big-endian format.
 421          * The fields should be presented as in big-endian format either.
 422          * Mask must be always present, it defines the actual field width.
 423          */
 424         MLX5_ASSERT(item->mask);
 425         MLX5_ASSERT(field->size);
 426         do {
 427                 uint32_t size_b;
 428                 uint32_t off_b;
 429                 uint32_t mask;
 430                 uint32_t data;
 431                 bool next_field = true;
 432                 bool next_dcopy = true;
 433
 434                 if (i >= MLX5_MAX_MODIFY_NUM)
 435                         return rte_flow_error_set(error, EINVAL,
 436                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 437                                  "too many items to modify");
 438                 /* Fetch variable byte size mask from the array. */
 439                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 440                                            field->offset, field->size);
 441                 if (!mask) {
 442                         ++field;
 443                         continue;
 444                 }
 445                 /* Deduce actual data width in bits from mask value. */
 446                 off_b = rte_bsf32(mask) + carry_b;
 447                 size_b = sizeof(uint32_t) * CHAR_BIT -
 448                          off_b - __builtin_clz(mask);
 449                 MLX5_ASSERT(size_b);
 450                 actions[i] = (struct mlx5_modification_cmd) {
 451                         .action_type = type,
 452                         .field = field->id,
 453                         .offset = off_b,
 454                         .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ?
 455                                 0 : size_b,
 456                 };
 457                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 458                         MLX5_ASSERT(dcopy);
 459                         actions[i].dst_field = dcopy->id;
 460                         actions[i].dst_offset =
 461                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 462                         /* Convert entire record to big-endian format. */
 463                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 464                         /*
 465                          * Destination field overflow. Copy leftovers of
 466                          * a source field to the next destination field.
 467                          */
 468                         carry_b = 0;
 469                         if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) &&
 470                             dcopy->size != 0) {
 471                                 actions[i].length =
 472                                         dcopy->size * CHAR_BIT - dcopy->offset;
 473                                 carry_b = actions[i].length;
 474                                 next_field = false;
 475                         }
 476                         /*
 477                          * Not enough bits in a source filed to fill a
 478                          * destination field. Switch to the next source.
 479                          */
 480                         if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) &&
 481                             (size_b == field->size * CHAR_BIT - off_b)) {
 482                                 actions[i].length =
 483                                         field->size * CHAR_BIT - off_b;
 484                                 dcopy->offset += actions[i].length;
 485                                 next_dcopy = false;
 486                         }
 487                         if (next_dcopy)
 488                                 ++dcopy;
 489                 } else {
 490                         MLX5_ASSERT(item->spec);
 491                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 492                                                    field->offset, field->size);
 493                         /* Shift out the trailing masked bits from data. */
 494                         data = (data & mask) >> off_b;
 495                         actions[i].data1 = rte_cpu_to_be_32(data);
 496                 }
 497                 /* Convert entire record to expected big-endian format. */
 498                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 499                 if (next_field)
 500                         ++field;
 501                 ++i;
 502         } while (field->size);
 503         if (resource->actions_num == i)
 504                 return rte_flow_error_set(error, EINVAL,
 505                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 506                                           "invalid modification flow item");
 507         resource->actions_num = i;
 508         return 0;
 509 }
 510
 511 /**
 512  * Convert modify-header set IPv4 address action to DV specification.
 513  *
 514  * @param[in,out] resource
 515  *   Pointer to the modify-header resource.
 516  * @param[in] action
 517  *   Pointer to action specification.
 518  * @param[out] error
 519  *   Pointer to the error structure.
 520  *
 521  * @return
 522  *   0 on success, a negative errno value otherwise and rte_errno is set.
 523  */
 524 static int
 525 flow_dv_convert_action_modify_ipv4
 526                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 527                          const struct rte_flow_action *action,
 528                          struct rte_flow_error *error)
 529 {
 530         const struct rte_flow_action_set_ipv4 *conf =
 531                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 532         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 533         struct rte_flow_item_ipv4 ipv4;
 534         struct rte_flow_item_ipv4 ipv4_mask;
 535
 536         memset(&ipv4, 0, sizeof(ipv4));
 537         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 538         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 539                 ipv4.hdr.src_addr = conf->ipv4_addr;
 540                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 541         } else {
 542                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 543                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 544         }
 545         item.spec = &ipv4;
 546         item.mask = &ipv4_mask;
 547         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 548                                              MLX5_MODIFICATION_TYPE_SET, error);
 549 }
 550
 551 /**
 552  * Convert modify-header set IPv6 address action to DV specification.
 553  *
 554  * @param[in,out] resource
 555  *   Pointer to the modify-header resource.
 556  * @param[in] action
 557  *   Pointer to action specification.
 558  * @param[out] error
 559  *   Pointer to the error structure.
 560  *
 561  * @return
 562  *   0 on success, a negative errno value otherwise and rte_errno is set.
 563  */
 564 static int
 565 flow_dv_convert_action_modify_ipv6
 566                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 567                          const struct rte_flow_action *action,
 568                          struct rte_flow_error *error)
 569 {
 570         const struct rte_flow_action_set_ipv6 *conf =
 571                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 572         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 573         struct rte_flow_item_ipv6 ipv6;
 574         struct rte_flow_item_ipv6 ipv6_mask;
 575
 576         memset(&ipv6, 0, sizeof(ipv6));
 577         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 578         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 579                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 580                        sizeof(ipv6.hdr.src_addr));
 581                 memcpy(&ipv6_mask.hdr.src_addr,
 582                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 583                        sizeof(ipv6.hdr.src_addr));
 584         } else {
 585                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 586                        sizeof(ipv6.hdr.dst_addr));
 587                 memcpy(&ipv6_mask.hdr.dst_addr,
 588                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 589                        sizeof(ipv6.hdr.dst_addr));
 590         }
 591         item.spec = &ipv6;
 592         item.mask = &ipv6_mask;
 593         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 594                                              MLX5_MODIFICATION_TYPE_SET, error);
 595 }
 596
 597 /**
 598  * Convert modify-header set MAC address action to DV specification.
 599  *
 600  * @param[in,out] resource
 601  *   Pointer to the modify-header resource.
 602  * @param[in] action
 603  *   Pointer to action specification.
 604  * @param[out] error
 605  *   Pointer to the error structure.
 606  *
 607  * @return
 608  *   0 on success, a negative errno value otherwise and rte_errno is set.
 609  */
 610 static int
 611 flow_dv_convert_action_modify_mac
 612                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 613                          const struct rte_flow_action *action,
 614                          struct rte_flow_error *error)
 615 {
 616         const struct rte_flow_action_set_mac *conf =
 617                 (const struct rte_flow_action_set_mac *)(action->conf);
 618         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 619         struct rte_flow_item_eth eth;
 620         struct rte_flow_item_eth eth_mask;
 621
 622         memset(&eth, 0, sizeof(eth));
 623         memset(&eth_mask, 0, sizeof(eth_mask));
 624         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 625                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 626                        sizeof(eth.src.addr_bytes));
 627                 memcpy(&eth_mask.src.addr_bytes,
 628                        &rte_flow_item_eth_mask.src.addr_bytes,
 629                        sizeof(eth_mask.src.addr_bytes));
 630         } else {
 631                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 632                        sizeof(eth.dst.addr_bytes));
 633                 memcpy(&eth_mask.dst.addr_bytes,
 634                        &rte_flow_item_eth_mask.dst.addr_bytes,
 635                        sizeof(eth_mask.dst.addr_bytes));
 636         }
 637         item.spec = &eth;
 638         item.mask = &eth_mask;
 639         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 640                                              MLX5_MODIFICATION_TYPE_SET, error);
 641 }
 642
 643 /**
 644  * Convert modify-header set VLAN VID action to DV specification.
 645  *
 646  * @param[in,out] resource
 647  *   Pointer to the modify-header resource.
 648  * @param[in] action
 649  *   Pointer to action specification.
 650  * @param[out] error
 651  *   Pointer to the error structure.
 652  *
 653  * @return
 654  *   0 on success, a negative errno value otherwise and rte_errno is set.
 655  */
 656 static int
 657 flow_dv_convert_action_modify_vlan_vid
 658                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 659                          const struct rte_flow_action *action,
 660                          struct rte_flow_error *error)
 661 {
 662         const struct rte_flow_action_of_set_vlan_vid *conf =
 663                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 664         int i = resource->actions_num;
 665         struct mlx5_modification_cmd *actions = resource->actions;
 666         struct field_modify_info *field = modify_vlan_out_first_vid;
 667
 668         if (i >= MLX5_MAX_MODIFY_NUM)
 669                 return rte_flow_error_set(error, EINVAL,
 670                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 671                          "too many items to modify");
 672         actions[i] = (struct mlx5_modification_cmd) {
 673                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 674                 .field = field->id,
 675                 .length = field->size,
 676                 .offset = field->offset,
 677         };
 678         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 679         actions[i].data1 = conf->vlan_vid;
 680         actions[i].data1 = actions[i].data1 << 16;
 681         resource->actions_num = ++i;
 682         return 0;
 683 }
 684
 685 /**
 686  * Convert modify-header set TP action to DV specification.
 687  *
 688  * @param[in,out] resource
 689  *   Pointer to the modify-header resource.
 690  * @param[in] action
 691  *   Pointer to action specification.
 692  * @param[in] items
 693  *   Pointer to rte_flow_item objects list.
 694  * @param[in] attr
 695  *   Pointer to flow attributes structure.
 696  * @param[in] dev_flow
 697  *   Pointer to the sub flow.
 698  * @param[in] tunnel_decap
 699  *   Whether action is after tunnel decapsulation.
 700  * @param[out] error
 701  *   Pointer to the error structure.
 702  *
 703  * @return
 704  *   0 on success, a negative errno value otherwise and rte_errno is set.
 705  */
 706 static int
 707 flow_dv_convert_action_modify_tp
 708                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 709                          const struct rte_flow_action *action,
 710                          const struct rte_flow_item *items,
 711                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 712                          bool tunnel_decap, struct rte_flow_error *error)
 713 {
 714         const struct rte_flow_action_set_tp *conf =
 715                 (const struct rte_flow_action_set_tp *)(action->conf);
 716         struct rte_flow_item item;
 717         struct rte_flow_item_udp udp;
 718         struct rte_flow_item_udp udp_mask;
 719         struct rte_flow_item_tcp tcp;
 720         struct rte_flow_item_tcp tcp_mask;
 721         struct field_modify_info *field;
 722
 723         if (!attr->valid)
 724                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 725         if (attr->udp) {
 726                 memset(&udp, 0, sizeof(udp));
 727                 memset(&udp_mask, 0, sizeof(udp_mask));
 728                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 729                         udp.hdr.src_port = conf->port;
 730                         udp_mask.hdr.src_port =
 731                                         rte_flow_item_udp_mask.hdr.src_port;
 732                 } else {
 733                         udp.hdr.dst_port = conf->port;
 734                         udp_mask.hdr.dst_port =
 735                                         rte_flow_item_udp_mask.hdr.dst_port;
 736                 }
 737                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 738                 item.spec = &udp;
 739                 item.mask = &udp_mask;
 740                 field = modify_udp;
 741         } else {
 742                 MLX5_ASSERT(attr->tcp);
 743                 memset(&tcp, 0, sizeof(tcp));
 744                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 745                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 746                         tcp.hdr.src_port = conf->port;
 747                         tcp_mask.hdr.src_port =
 748                                         rte_flow_item_tcp_mask.hdr.src_port;
 749                 } else {
 750                         tcp.hdr.dst_port = conf->port;
 751                         tcp_mask.hdr.dst_port =
 752                                         rte_flow_item_tcp_mask.hdr.dst_port;
 753                 }
 754                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 755                 item.spec = &tcp;
 756                 item.mask = &tcp_mask;
 757                 field = modify_tcp;
 758         }
 759         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 760                                              MLX5_MODIFICATION_TYPE_SET, error);
 761 }
 762
 763 /**
 764  * Convert modify-header set TTL action to DV specification.
 765  *
 766  * @param[in,out] resource
 767  *   Pointer to the modify-header resource.
 768  * @param[in] action
 769  *   Pointer to action specification.
 770  * @param[in] items
 771  *   Pointer to rte_flow_item objects list.
 772  * @param[in] attr
 773  *   Pointer to flow attributes structure.
 774  * @param[in] dev_flow
 775  *   Pointer to the sub flow.
 776  * @param[in] tunnel_decap
 777  *   Whether action is after tunnel decapsulation.
 778  * @param[out] error
 779  *   Pointer to the error structure.
 780  *
 781  * @return
 782  *   0 on success, a negative errno value otherwise and rte_errno is set.
 783  */
 784 static int
 785 flow_dv_convert_action_modify_ttl
 786                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 787                          const struct rte_flow_action *action,
 788                          const struct rte_flow_item *items,
 789                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 790                          bool tunnel_decap, struct rte_flow_error *error)
 791 {
 792         const struct rte_flow_action_set_ttl *conf =
 793                 (const struct rte_flow_action_set_ttl *)(action->conf);
 794         struct rte_flow_item item;
 795         struct rte_flow_item_ipv4 ipv4;
 796         struct rte_flow_item_ipv4 ipv4_mask;
 797         struct rte_flow_item_ipv6 ipv6;
 798         struct rte_flow_item_ipv6 ipv6_mask;
 799         struct field_modify_info *field;
 800
 801         if (!attr->valid)
 802                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 803         if (attr->ipv4) {
 804                 memset(&ipv4, 0, sizeof(ipv4));
 805                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 806                 ipv4.hdr.time_to_live = conf->ttl_value;
 807                 ipv4_mask.hdr.time_to_live = 0xFF;
 808                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 809                 item.spec = &ipv4;
 810                 item.mask = &ipv4_mask;
 811                 field = modify_ipv4;
 812         } else {
 813                 MLX5_ASSERT(attr->ipv6);
 814                 memset(&ipv6, 0, sizeof(ipv6));
 815                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 816                 ipv6.hdr.hop_limits = conf->ttl_value;
 817                 ipv6_mask.hdr.hop_limits = 0xFF;
 818                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 819                 item.spec = &ipv6;
 820                 item.mask = &ipv6_mask;
 821                 field = modify_ipv6;
 822         }
 823         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 824                                              MLX5_MODIFICATION_TYPE_SET, error);
 825 }
 826
 827 /**
 828  * Convert modify-header decrement TTL action to DV specification.
 829  *
 830  * @param[in,out] resource
 831  *   Pointer to the modify-header resource.
 832  * @param[in] action
 833  *   Pointer to action specification.
 834  * @param[in] items
 835  *   Pointer to rte_flow_item objects list.
 836  * @param[in] attr
 837  *   Pointer to flow attributes structure.
 838  * @param[in] dev_flow
 839  *   Pointer to the sub flow.
 840  * @param[in] tunnel_decap
 841  *   Whether action is after tunnel decapsulation.
 842  * @param[out] error
 843  *   Pointer to the error structure.
 844  *
 845  * @return
 846  *   0 on success, a negative errno value otherwise and rte_errno is set.
 847  */
 848 static int
 849 flow_dv_convert_action_modify_dec_ttl
 850                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 851                          const struct rte_flow_item *items,
 852                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 853                          bool tunnel_decap, struct rte_flow_error *error)
 854 {
 855         struct rte_flow_item item;
 856         struct rte_flow_item_ipv4 ipv4;
 857         struct rte_flow_item_ipv4 ipv4_mask;
 858         struct rte_flow_item_ipv6 ipv6;
 859         struct rte_flow_item_ipv6 ipv6_mask;
 860         struct field_modify_info *field;
 861
 862         if (!attr->valid)
 863                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 864         if (attr->ipv4) {
 865                 memset(&ipv4, 0, sizeof(ipv4));
 866                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 867                 ipv4.hdr.time_to_live = 0xFF;
 868                 ipv4_mask.hdr.time_to_live = 0xFF;
 869                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 870                 item.spec = &ipv4;
 871                 item.mask = &ipv4_mask;
 872                 field = modify_ipv4;
 873         } else {
 874                 MLX5_ASSERT(attr->ipv6);
 875                 memset(&ipv6, 0, sizeof(ipv6));
 876                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 877                 ipv6.hdr.hop_limits = 0xFF;
 878                 ipv6_mask.hdr.hop_limits = 0xFF;
 879                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 880                 item.spec = &ipv6;
 881                 item.mask = &ipv6_mask;
 882                 field = modify_ipv6;
 883         }
 884         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 885                                              MLX5_MODIFICATION_TYPE_ADD, error);
 886 }
 887
 888 /**
 889  * Convert modify-header increment/decrement TCP Sequence number
 890  * to DV specification.
 891  *
 892  * @param[in,out] resource
 893  *   Pointer to the modify-header resource.
 894  * @param[in] action
 895  *   Pointer to action specification.
 896  * @param[out] error
 897  *   Pointer to the error structure.
 898  *
 899  * @return
 900  *   0 on success, a negative errno value otherwise and rte_errno is set.
 901  */
 902 static int
 903 flow_dv_convert_action_modify_tcp_seq
 904                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 905                          const struct rte_flow_action *action,
 906                          struct rte_flow_error *error)
 907 {
 908         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 909         uint64_t value = rte_be_to_cpu_32(*conf);
 910         struct rte_flow_item item;
 911         struct rte_flow_item_tcp tcp;
 912         struct rte_flow_item_tcp tcp_mask;
 913
 914         memset(&tcp, 0, sizeof(tcp));
 915         memset(&tcp_mask, 0, sizeof(tcp_mask));
 916         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 917                 /*
 918                  * The HW has no decrement operation, only increment operation.
 919                  * To simulate decrement X from Y using increment operation
 920                  * we need to add UINT32_MAX X times to Y.
 921                  * Each adding of UINT32_MAX decrements Y by 1.
 922                  */
 923                 value *= UINT32_MAX;
 924         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 925         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 926         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 927         item.spec = &tcp;
 928         item.mask = &tcp_mask;
 929         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 930                                              MLX5_MODIFICATION_TYPE_ADD, error);
 931 }
 932
 933 /**
 934  * Convert modify-header increment/decrement TCP Acknowledgment number
 935  * to DV specification.
 936  *
 937  * @param[in,out] resource
 938  *   Pointer to the modify-header resource.
 939  * @param[in] action
 940  *   Pointer to action specification.
 941  * @param[out] error
 942  *   Pointer to the error structure.
 943  *
 944  * @return
 945  *   0 on success, a negative errno value otherwise and rte_errno is set.
 946  */
 947 static int
 948 flow_dv_convert_action_modify_tcp_ack
 949                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 950                          const struct rte_flow_action *action,
 951                          struct rte_flow_error *error)
 952 {
 953         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 954         uint64_t value = rte_be_to_cpu_32(*conf);
 955         struct rte_flow_item item;
 956         struct rte_flow_item_tcp tcp;
 957         struct rte_flow_item_tcp tcp_mask;
 958
 959         memset(&tcp, 0, sizeof(tcp));
 960         memset(&tcp_mask, 0, sizeof(tcp_mask));
 961         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 962                 /*
 963                  * The HW has no decrement operation, only increment operation.
 964                  * To simulate decrement X from Y using increment operation
 965                  * we need to add UINT32_MAX X times to Y.
 966                  * Each adding of UINT32_MAX decrements Y by 1.
 967                  */
 968                 value *= UINT32_MAX;
 969         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 970         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 971         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 972         item.spec = &tcp;
 973         item.mask = &tcp_mask;
 974         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 975                                              MLX5_MODIFICATION_TYPE_ADD, error);
 976 }
 977
 978 static enum mlx5_modification_field reg_to_field[] = {
 979         [REG_NON] = MLX5_MODI_OUT_NONE,
 980         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 981         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 982         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 983         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 984         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 985         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 986         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 987         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 988         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 989         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 990 };
 991
 992 /**
 993  * Convert register set to DV specification.
 994  *
 995  * @param[in,out] resource
 996  *   Pointer to the modify-header resource.
 997  * @param[in] action
 998  *   Pointer to action specification.
 999  * @param[out] error
1000  *   Pointer to the error structure.
1001  *
1002  * @return
1003  *   0 on success, a negative errno value otherwise and rte_errno is set.
1004  */
1005 static int
1006 flow_dv_convert_action_set_reg
1007                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1008                          const struct rte_flow_action *action,
1009                          struct rte_flow_error *error)
1010 {
1011         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
1012         struct mlx5_modification_cmd *actions = resource->actions;
1013         uint32_t i = resource->actions_num;
1014
1015         if (i >= MLX5_MAX_MODIFY_NUM)
1016                 return rte_flow_error_set(error, EINVAL,
1017                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1018                                           "too many items to modify");
1019         MLX5_ASSERT(conf->id != REG_NON);
1020         MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
1021         actions[i] = (struct mlx5_modification_cmd) {
1022                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1023                 .field = reg_to_field[conf->id],
1024                 .offset = conf->offset,
1025                 .length = conf->length,
1026         };
1027         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1028         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1029         ++i;
1030         resource->actions_num = i;
1031         return 0;
1032 }
1033
1034 /**
1035  * Convert SET_TAG action to DV specification.
1036  *
1037  * @param[in] dev
1038  *   Pointer to the rte_eth_dev structure.
1039  * @param[in,out] resource
1040  *   Pointer to the modify-header resource.
1041  * @param[in] conf
1042  *   Pointer to action specification.
1043  * @param[out] error
1044  *   Pointer to the error structure.
1045  *
1046  * @return
1047  *   0 on success, a negative errno value otherwise and rte_errno is set.
1048  */
1049 static int
1050 flow_dv_convert_action_set_tag
1051                         (struct rte_eth_dev *dev,
1052                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1053                          const struct rte_flow_action_set_tag *conf,
1054                          struct rte_flow_error *error)
1055 {
1056         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1057         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1058         struct rte_flow_item item = {
1059                 .spec = &data,
1060                 .mask = &mask,
1061         };
1062         struct field_modify_info reg_c_x[] = {
1063                 [1] = {0, 0, 0},
1064         };
1065         enum mlx5_modification_field reg_type;
1066         int ret;
1067
1068         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1069         if (ret < 0)
1070                 return ret;
1071         MLX5_ASSERT(ret != REG_NON);
1072         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1073         reg_type = reg_to_field[ret];
1074         MLX5_ASSERT(reg_type > 0);
1075         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1076         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1077                                              MLX5_MODIFICATION_TYPE_SET, error);
1078 }
1079
1080 /**
1081  * Convert internal COPY_REG action to DV specification.
1082  *
1083  * @param[in] dev
1084  *   Pointer to the rte_eth_dev structure.
1085  * @param[in,out] res
1086  *   Pointer to the modify-header resource.
1087  * @param[in] action
1088  *   Pointer to action specification.
1089  * @param[out] error
1090  *   Pointer to the error structure.
1091  *
1092  * @return
1093  *   0 on success, a negative errno value otherwise and rte_errno is set.
1094  */
1095 static int
1096 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1097                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1098                                  const struct rte_flow_action *action,
1099                                  struct rte_flow_error *error)
1100 {
1101         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1102         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1103         struct rte_flow_item item = {
1104                 .spec = NULL,
1105                 .mask = &mask,
1106         };
1107         struct field_modify_info reg_src[] = {
1108                 {4, 0, reg_to_field[conf->src]},
1109                 {0, 0, 0},
1110         };
1111         struct field_modify_info reg_dst = {
1112                 .offset = 0,
1113                 .id = reg_to_field[conf->dst],
1114         };
1115         /* Adjust reg_c[0] usage according to reported mask. */
1116         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1117                 struct mlx5_priv *priv = dev->data->dev_private;
1118                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1119
1120                 MLX5_ASSERT(reg_c0);
1121                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1122                 if (conf->dst == REG_C_0) {
1123                         /* Copy to reg_c[0], within mask only. */
1124                         reg_dst.offset = rte_bsf32(reg_c0);
1125                         /*
1126                          * Mask is ignoring the enianness, because
1127                          * there is no conversion in datapath.
1128                          */
1129 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1130                         /* Copy from destination lower bits to reg_c[0]. */
1131                         mask = reg_c0 >> reg_dst.offset;
1132 #else
1133                         /* Copy from destination upper bits to reg_c[0]. */
1134                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1135                                           rte_fls_u32(reg_c0));
1136 #endif
1137                 } else {
1138                         mask = rte_cpu_to_be_32(reg_c0);
1139 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1140                         /* Copy from reg_c[0] to destination lower bits. */
1141                         reg_dst.offset = 0;
1142 #else
1143                         /* Copy from reg_c[0] to destination upper bits. */
1144                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1145                                          (rte_fls_u32(reg_c0) -
1146                                           rte_bsf32(reg_c0));
1147 #endif
1148                 }
1149         }
1150         return flow_dv_convert_modify_action(&item,
1151                                              reg_src, &reg_dst, res,
1152                                              MLX5_MODIFICATION_TYPE_COPY,
1153                                              error);
1154 }
1155
1156 /**
1157  * Convert MARK action to DV specification. This routine is used
1158  * in extensive metadata only and requires metadata register to be
1159  * handled. In legacy mode hardware tag resource is engaged.
1160  *
1161  * @param[in] dev
1162  *   Pointer to the rte_eth_dev structure.
1163  * @param[in] conf
1164  *   Pointer to MARK action specification.
1165  * @param[in,out] resource
1166  *   Pointer to the modify-header resource.
1167  * @param[out] error
1168  *   Pointer to the error structure.
1169  *
1170  * @return
1171  *   0 on success, a negative errno value otherwise and rte_errno is set.
1172  */
1173 static int
1174 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1175                             const struct rte_flow_action_mark *conf,
1176                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1177                             struct rte_flow_error *error)
1178 {
1179         struct mlx5_priv *priv = dev->data->dev_private;
1180         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1181                                            priv->sh->dv_mark_mask);
1182         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1183         struct rte_flow_item item = {
1184                 .spec = &data,
1185                 .mask = &mask,
1186         };
1187         struct field_modify_info reg_c_x[] = {
1188                 [1] = {0, 0, 0},
1189         };
1190         int reg;
1191
1192         if (!mask)
1193                 return rte_flow_error_set(error, EINVAL,
1194                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1195                                           NULL, "zero mark action mask");
1196         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1197         if (reg < 0)
1198                 return reg;
1199         MLX5_ASSERT(reg > 0);
1200         if (reg == REG_C_0) {
1201                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1202                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1203
1204                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1205                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1206                 mask = rte_cpu_to_be_32(mask << shl_c0);
1207         }
1208         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1209         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1210                                              MLX5_MODIFICATION_TYPE_SET, error);
1211 }
1212
1213 /**
1214  * Get metadata register index for specified steering domain.
1215  *
1216  * @param[in] dev
1217  *   Pointer to the rte_eth_dev structure.
1218  * @param[in] attr
1219  *   Attributes of flow to determine steering domain.
1220  * @param[out] error
1221  *   Pointer to the error structure.
1222  *
1223  * @return
1224  *   positive index on success, a negative errno value otherwise
1225  *   and rte_errno is set.
1226  */
1227 static enum modify_reg
1228 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1229                          const struct rte_flow_attr *attr,
1230                          struct rte_flow_error *error)
1231 {
1232         int reg =
1233                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1234                                           MLX5_METADATA_FDB :
1235                                             attr->egress ?
1236                                             MLX5_METADATA_TX :
1237                                             MLX5_METADATA_RX, 0, error);
1238         if (reg < 0)
1239                 return rte_flow_error_set(error,
1240                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1241                                           NULL, "unavailable "
1242                                           "metadata register");
1243         return reg;
1244 }
1245
1246 /**
1247  * Convert SET_META action to DV specification.
1248  *
1249  * @param[in] dev
1250  *   Pointer to the rte_eth_dev structure.
1251  * @param[in,out] resource
1252  *   Pointer to the modify-header resource.
1253  * @param[in] attr
1254  *   Attributes of flow that includes this item.
1255  * @param[in] conf
1256  *   Pointer to action specification.
1257  * @param[out] error
1258  *   Pointer to the error structure.
1259  *
1260  * @return
1261  *   0 on success, a negative errno value otherwise and rte_errno is set.
1262  */
1263 static int
1264 flow_dv_convert_action_set_meta
1265                         (struct rte_eth_dev *dev,
1266                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1267                          const struct rte_flow_attr *attr,
1268                          const struct rte_flow_action_set_meta *conf,
1269                          struct rte_flow_error *error)
1270 {
1271         uint32_t mask = rte_cpu_to_be_32(conf->mask);
1272         uint32_t data = rte_cpu_to_be_32(conf->data) & mask;
1273         struct rte_flow_item item = {
1274                 .spec = &data,
1275                 .mask = &mask,
1276         };
1277         struct field_modify_info reg_c_x[] = {
1278                 [1] = {0, 0, 0},
1279         };
1280         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1281
1282         if (reg < 0)
1283                 return reg;
1284         MLX5_ASSERT(reg != REG_NON);
1285         if (reg == REG_C_0) {
1286                 struct mlx5_priv *priv = dev->data->dev_private;
1287                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1288                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1289
1290                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1291                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1292                 mask = rte_cpu_to_be_32(mask << shl_c0);
1293         }
1294         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1295         /* The routine expects parameters in memory as big-endian ones. */
1296         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1297                                              MLX5_MODIFICATION_TYPE_SET, error);
1298 }
1299
1300 /**
1301  * Convert modify-header set IPv4 DSCP action to DV specification.
1302  *
1303  * @param[in,out] resource
1304  *   Pointer to the modify-header resource.
1305  * @param[in] action
1306  *   Pointer to action specification.
1307  * @param[out] error
1308  *   Pointer to the error structure.
1309  *
1310  * @return
1311  *   0 on success, a negative errno value otherwise and rte_errno is set.
1312  */
1313 static int
1314 flow_dv_convert_action_modify_ipv4_dscp
1315                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1316                          const struct rte_flow_action *action,
1317                          struct rte_flow_error *error)
1318 {
1319         const struct rte_flow_action_set_dscp *conf =
1320                 (const struct rte_flow_action_set_dscp *)(action->conf);
1321         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1322         struct rte_flow_item_ipv4 ipv4;
1323         struct rte_flow_item_ipv4 ipv4_mask;
1324
1325         memset(&ipv4, 0, sizeof(ipv4));
1326         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1327         ipv4.hdr.type_of_service = conf->dscp;
1328         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1329         item.spec = &ipv4;
1330         item.mask = &ipv4_mask;
1331         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1332                                              MLX5_MODIFICATION_TYPE_SET, error);
1333 }
1334
1335 /**
1336  * Convert modify-header set IPv6 DSCP action to DV specification.
1337  *
1338  * @param[in,out] resource
1339  *   Pointer to the modify-header resource.
1340  * @param[in] action
1341  *   Pointer to action specification.
1342  * @param[out] error
1343  *   Pointer to the error structure.
1344  *
1345  * @return
1346  *   0 on success, a negative errno value otherwise and rte_errno is set.
1347  */
1348 static int
1349 flow_dv_convert_action_modify_ipv6_dscp
1350                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1351                          const struct rte_flow_action *action,
1352                          struct rte_flow_error *error)
1353 {
1354         const struct rte_flow_action_set_dscp *conf =
1355                 (const struct rte_flow_action_set_dscp *)(action->conf);
1356         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1357         struct rte_flow_item_ipv6 ipv6;
1358         struct rte_flow_item_ipv6 ipv6_mask;
1359
1360         memset(&ipv6, 0, sizeof(ipv6));
1361         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1362         /*
1363          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1364          * rdma-core only accept the DSCP bits byte aligned start from
1365          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1366          * bits in IPv6 case as rdma-core requires byte aligned value.
1367          */
1368         ipv6.hdr.vtc_flow = conf->dscp;
1369         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1370         item.spec = &ipv6;
1371         item.mask = &ipv6_mask;
1372         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1373                                              MLX5_MODIFICATION_TYPE_SET, error);
1374 }
1375
1376 static int
1377 mlx5_flow_item_field_width(struct mlx5_dev_config *config,
1378                            enum rte_flow_field_id field)
1379 {
1380         switch (field) {
1381         case RTE_FLOW_FIELD_START:
1382                 return 32;
1383         case RTE_FLOW_FIELD_MAC_DST:
1384         case RTE_FLOW_FIELD_MAC_SRC:
1385                 return 48;
1386         case RTE_FLOW_FIELD_VLAN_TYPE:
1387                 return 16;
1388         case RTE_FLOW_FIELD_VLAN_ID:
1389                 return 12;
1390         case RTE_FLOW_FIELD_MAC_TYPE:
1391                 return 16;
1392         case RTE_FLOW_FIELD_IPV4_DSCP:
1393                 return 6;
1394         case RTE_FLOW_FIELD_IPV4_TTL:
1395                 return 8;
1396         case RTE_FLOW_FIELD_IPV4_SRC:
1397         case RTE_FLOW_FIELD_IPV4_DST:
1398                 return 32;
1399         case RTE_FLOW_FIELD_IPV6_DSCP:
1400                 return 6;
1401         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1402                 return 8;
1403         case RTE_FLOW_FIELD_IPV6_SRC:
1404         case RTE_FLOW_FIELD_IPV6_DST:
1405                 return 128;
1406         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1407         case RTE_FLOW_FIELD_TCP_PORT_DST:
1408                 return 16;
1409         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1410         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1411                 return 32;
1412         case RTE_FLOW_FIELD_TCP_FLAGS:
1413                 return 9;
1414         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1415         case RTE_FLOW_FIELD_UDP_PORT_DST:
1416                 return 16;
1417         case RTE_FLOW_FIELD_VXLAN_VNI:
1418         case RTE_FLOW_FIELD_GENEVE_VNI:
1419                 return 24;
1420         case RTE_FLOW_FIELD_GTP_TEID:
1421         case RTE_FLOW_FIELD_TAG:
1422                 return 32;
1423         case RTE_FLOW_FIELD_MARK:
1424                 return 24;
1425         case RTE_FLOW_FIELD_META:
1426                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_META16)
1427                         return 16;
1428                 else if (config->dv_xmeta_en == MLX5_XMETA_MODE_META32)
1429                         return 32;
1430                 else
1431                         return 0;
1432         case RTE_FLOW_FIELD_POINTER:
1433         case RTE_FLOW_FIELD_VALUE:
1434                 return 64;
1435         default:
1436                 MLX5_ASSERT(false);
1437         }
1438         return 0;
1439 }
1440
1441 static void
1442 mlx5_flow_field_id_to_modify_info
1443                 (const struct rte_flow_action_modify_data *data,
1444                  struct field_modify_info *info,
1445                  uint32_t *mask, uint32_t *value,
1446                  uint32_t width, uint32_t dst_width,
1447                  struct rte_eth_dev *dev,
1448                  const struct rte_flow_attr *attr,
1449                  struct rte_flow_error *error)
1450 {
1451         struct mlx5_priv *priv = dev->data->dev_private;
1452         struct mlx5_dev_config *config = &priv->config;
1453         uint32_t idx = 0;
1454         uint32_t off = 0;
1455         uint64_t val = 0;
1456         switch (data->field) {
1457         case RTE_FLOW_FIELD_START:
1458                 /* not supported yet */
1459                 MLX5_ASSERT(false);
1460                 break;
1461         case RTE_FLOW_FIELD_MAC_DST:
1462                 off = data->offset > 16 ? data->offset - 16 : 0;
1463                 if (mask) {
1464                         if (data->offset < 16) {
1465                                 info[idx] = (struct field_modify_info){2, 0,
1466                                                 MLX5_MODI_OUT_DMAC_15_0};
1467                                 if (width < 16) {
1468                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1469                                                                  (16 - width));
1470                                         width = 0;
1471                                 } else {
1472                                         mask[idx] = RTE_BE16(0xffff);
1473                                         width -= 16;
1474                                 }
1475                                 if (!width)
1476                                         break;
1477                                 ++idx;
1478                         }
1479                         info[idx] = (struct field_modify_info){4, 4 * idx,
1480                                                 MLX5_MODI_OUT_DMAC_47_16};
1481                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1482                                                       (32 - width)) << off);
1483                 } else {
1484                         if (data->offset < 16)
1485                                 info[idx++] = (struct field_modify_info){2, 0,
1486                                                 MLX5_MODI_OUT_DMAC_15_0};
1487                         info[idx] = (struct field_modify_info){4, off,
1488                                                 MLX5_MODI_OUT_DMAC_47_16};
1489                 }
1490                 break;
1491         case RTE_FLOW_FIELD_MAC_SRC:
1492                 off = data->offset > 16 ? data->offset - 16 : 0;
1493                 if (mask) {
1494                         if (data->offset < 16) {
1495                                 info[idx] = (struct field_modify_info){2, 0,
1496                                                 MLX5_MODI_OUT_SMAC_15_0};
1497                                 if (width < 16) {
1498                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1499                                                                  (16 - width));
1500                                         width = 0;
1501                                 } else {
1502                                         mask[idx] = RTE_BE16(0xffff);
1503                                         width -= 16;
1504                                 }
1505                                 if (!width)
1506                                         break;
1507                                 ++idx;
1508                         }
1509                         info[idx] = (struct field_modify_info){4, 4 * idx,
1510                                                 MLX5_MODI_OUT_SMAC_47_16};
1511                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1512                                                       (32 - width)) << off);
1513                 } else {
1514                         if (data->offset < 16)
1515                                 info[idx++] = (struct field_modify_info){2, 0,
1516                                                 MLX5_MODI_OUT_SMAC_15_0};
1517                         info[idx] = (struct field_modify_info){4, off,
1518                                                 MLX5_MODI_OUT_SMAC_47_16};
1519                 }
1520                 break;
1521         case RTE_FLOW_FIELD_VLAN_TYPE:
1522                 /* not supported yet */
1523                 break;
1524         case RTE_FLOW_FIELD_VLAN_ID:
1525                 info[idx] = (struct field_modify_info){2, 0,
1526                                         MLX5_MODI_OUT_FIRST_VID};
1527                 if (mask)
1528                         mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
1529                 break;
1530         case RTE_FLOW_FIELD_MAC_TYPE:
1531                 info[idx] = (struct field_modify_info){2, 0,
1532                                         MLX5_MODI_OUT_ETHERTYPE};
1533                 if (mask)
1534                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1535                 break;
1536         case RTE_FLOW_FIELD_IPV4_DSCP:
1537                 info[idx] = (struct field_modify_info){1, 0,
1538                                         MLX5_MODI_OUT_IP_DSCP};
1539                 if (mask)
1540                         mask[idx] = 0x3f >> (6 - width);
1541                 break;
1542         case RTE_FLOW_FIELD_IPV4_TTL:
1543                 info[idx] = (struct field_modify_info){1, 0,
1544                                         MLX5_MODI_OUT_IPV4_TTL};
1545                 if (mask)
1546                         mask[idx] = 0xff >> (8 - width);
1547                 break;
1548         case RTE_FLOW_FIELD_IPV4_SRC:
1549                 info[idx] = (struct field_modify_info){4, 0,
1550                                         MLX5_MODI_OUT_SIPV4};
1551                 if (mask)
1552                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1553                                                      (32 - width));
1554                 break;
1555         case RTE_FLOW_FIELD_IPV4_DST:
1556                 info[idx] = (struct field_modify_info){4, 0,
1557                                         MLX5_MODI_OUT_DIPV4};
1558                 if (mask)
1559                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1560                                                      (32 - width));
1561                 break;
1562         case RTE_FLOW_FIELD_IPV6_DSCP:
1563                 info[idx] = (struct field_modify_info){1, 0,
1564                                         MLX5_MODI_OUT_IP_DSCP};
1565                 if (mask)
1566                         mask[idx] = 0x3f >> (6 - width);
1567                 break;
1568         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1569                 info[idx] = (struct field_modify_info){1, 0,
1570                                         MLX5_MODI_OUT_IPV6_HOPLIMIT};
1571                 if (mask)
1572                         mask[idx] = 0xff >> (8 - width);
1573                 break;
1574         case RTE_FLOW_FIELD_IPV6_SRC:
1575                 if (mask) {
1576                         if (data->offset < 32) {
1577                                 info[idx] = (struct field_modify_info){4,
1578                                                 4 * idx,
1579                                                 MLX5_MODI_OUT_SIPV6_31_0};
1580                                 if (width < 32) {
1581                                         mask[idx] =
1582                                                 rte_cpu_to_be_32(0xffffffff >>
1583                                                                  (32 - width));
1584                                         width = 0;
1585                                 } else {
1586                                         mask[idx] = RTE_BE32(0xffffffff);
1587                                         width -= 32;
1588                                 }
1589                                 if (!width)
1590                                         break;
1591                                 ++idx;
1592                         }
1593                         if (data->offset < 64) {
1594                                 info[idx] = (struct field_modify_info){4,
1595                                                 4 * idx,
1596                                                 MLX5_MODI_OUT_SIPV6_63_32};
1597                                 if (width < 32) {
1598                                         mask[idx] =
1599                                                 rte_cpu_to_be_32(0xffffffff >>
1600                                                                  (32 - width));
1601                                         width = 0;
1602                                 } else {
1603                                         mask[idx] = RTE_BE32(0xffffffff);
1604                                         width -= 32;
1605                                 }
1606                                 if (!width)
1607                                         break;
1608                                 ++idx;
1609                         }
1610                         if (data->offset < 96) {
1611                                 info[idx] = (struct field_modify_info){4,
1612                                                 4 * idx,
1613                                                 MLX5_MODI_OUT_SIPV6_95_64};
1614                                 if (width < 32) {
1615                                         mask[idx] =
1616                                                 rte_cpu_to_be_32(0xffffffff >>
1617                                                                  (32 - width));
1618                                         width = 0;
1619                                 } else {
1620                                         mask[idx] = RTE_BE32(0xffffffff);
1621                                         width -= 32;
1622                                 }
1623                                 if (!width)
1624                                         break;
1625                                 ++idx;
1626                         }
1627                         info[idx] = (struct field_modify_info){4, 4 * idx,
1628                                                 MLX5_MODI_OUT_SIPV6_127_96};
1629                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1630                                                      (32 - width));
1631                 } else {
1632                         if (data->offset < 32)
1633                                 info[idx++] = (struct field_modify_info){4, 0,
1634                                                 MLX5_MODI_OUT_SIPV6_31_0};
1635                         if (data->offset < 64)
1636                                 info[idx++] = (struct field_modify_info){4, 0,
1637                                                 MLX5_MODI_OUT_SIPV6_63_32};
1638                         if (data->offset < 96)
1639                                 info[idx++] = (struct field_modify_info){4, 0,
1640                                                 MLX5_MODI_OUT_SIPV6_95_64};
1641                         if (data->offset < 128)
1642                                 info[idx++] = (struct field_modify_info){4, 0,
1643                                                 MLX5_MODI_OUT_SIPV6_127_96};
1644                 }
1645                 break;
1646         case RTE_FLOW_FIELD_IPV6_DST:
1647                 if (mask) {
1648                         if (data->offset < 32) {
1649                                 info[idx] = (struct field_modify_info){4,
1650                                                 4 * idx,
1651                                                 MLX5_MODI_OUT_DIPV6_31_0};
1652                                 if (width < 32) {
1653                                         mask[idx] =
1654                                                 rte_cpu_to_be_32(0xffffffff >>
1655                                                                  (32 - width));
1656                                         width = 0;
1657                                 } else {
1658                                         mask[idx] = RTE_BE32(0xffffffff);
1659                                         width -= 32;
1660                                 }
1661                                 if (!width)
1662                                         break;
1663                                 ++idx;
1664                         }
1665                         if (data->offset < 64) {
1666                                 info[idx] = (struct field_modify_info){4,
1667                                                 4 * idx,
1668                                                 MLX5_MODI_OUT_DIPV6_63_32};
1669                                 if (width < 32) {
1670                                         mask[idx] =
1671                                                 rte_cpu_to_be_32(0xffffffff >>
1672                                                                  (32 - width));
1673                                         width = 0;
1674                                 } else {
1675                                         mask[idx] = RTE_BE32(0xffffffff);
1676                                         width -= 32;
1677                                 }
1678                                 if (!width)
1679                                         break;
1680                                 ++idx;
1681                         }
1682                         if (data->offset < 96) {
1683                                 info[idx] = (struct field_modify_info){4,
1684                                                 4 * idx,
1685                                                 MLX5_MODI_OUT_DIPV6_95_64};
1686                                 if (width < 32) {
1687                                         mask[idx] =
1688                                                 rte_cpu_to_be_32(0xffffffff >>
1689                                                                  (32 - width));
1690                                         width = 0;
1691                                 } else {
1692                                         mask[idx] = RTE_BE32(0xffffffff);
1693                                         width -= 32;
1694                                 }
1695                                 if (!width)
1696                                         break;
1697                                 ++idx;
1698                         }
1699                         info[idx] = (struct field_modify_info){4, 4 * idx,
1700                                                 MLX5_MODI_OUT_DIPV6_127_96};
1701                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1702                                                      (32 - width));
1703                 } else {
1704                         if (data->offset < 32)
1705                                 info[idx++] = (struct field_modify_info){4, 0,
1706                                                 MLX5_MODI_OUT_DIPV6_31_0};
1707                         if (data->offset < 64)
1708                                 info[idx++] = (struct field_modify_info){4, 0,
1709                                                 MLX5_MODI_OUT_DIPV6_63_32};
1710                         if (data->offset < 96)
1711                                 info[idx++] = (struct field_modify_info){4, 0,
1712                                                 MLX5_MODI_OUT_DIPV6_95_64};
1713                         if (data->offset < 128)
1714                                 info[idx++] = (struct field_modify_info){4, 0,
1715                                                 MLX5_MODI_OUT_DIPV6_127_96};
1716                 }
1717                 break;
1718         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1719                 info[idx] = (struct field_modify_info){2, 0,
1720                                         MLX5_MODI_OUT_TCP_SPORT};
1721                 if (mask)
1722                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1723                 break;
1724         case RTE_FLOW_FIELD_TCP_PORT_DST:
1725                 info[idx] = (struct field_modify_info){2, 0,
1726                                         MLX5_MODI_OUT_TCP_DPORT};
1727                 if (mask)
1728                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1729                 break;
1730         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1731                 info[idx] = (struct field_modify_info){4, 0,
1732                                         MLX5_MODI_OUT_TCP_SEQ_NUM};
1733                 if (mask)
1734                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1735                                                      (32 - width));
1736                 break;
1737         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1738                 info[idx] = (struct field_modify_info){4, 0,
1739                                         MLX5_MODI_OUT_TCP_ACK_NUM};
1740                 if (mask)
1741                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1742                                                      (32 - width));
1743                 break;
1744         case RTE_FLOW_FIELD_TCP_FLAGS:
1745                 info[idx] = (struct field_modify_info){2, 0,
1746                                         MLX5_MODI_OUT_TCP_FLAGS};
1747                 if (mask)
1748                         mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
1749                 break;
1750         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1751                 info[idx] = (struct field_modify_info){2, 0,
1752                                         MLX5_MODI_OUT_UDP_SPORT};
1753                 if (mask)
1754                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1755                 break;
1756         case RTE_FLOW_FIELD_UDP_PORT_DST:
1757                 info[idx] = (struct field_modify_info){2, 0,
1758                                         MLX5_MODI_OUT_UDP_DPORT};
1759                 if (mask)
1760                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1761                 break;
1762         case RTE_FLOW_FIELD_VXLAN_VNI:
1763                 /* not supported yet */
1764                 break;
1765         case RTE_FLOW_FIELD_GENEVE_VNI:
1766                 /* not supported yet*/
1767                 break;
1768         case RTE_FLOW_FIELD_GTP_TEID:
1769                 info[idx] = (struct field_modify_info){4, 0,
1770                                         MLX5_MODI_GTP_TEID};
1771                 if (mask)
1772                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1773                                                      (32 - width));
1774                 break;
1775         case RTE_FLOW_FIELD_TAG:
1776                 {
1777                         int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
1778                                                    data->level, error);
1779                         if (reg < 0)
1780                                 return;
1781                         MLX5_ASSERT(reg != REG_NON);
1782                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1783                         info[idx] = (struct field_modify_info){4, 0,
1784                                                 reg_to_field[reg]};
1785                         if (mask)
1786                                 mask[idx] =
1787                                         rte_cpu_to_be_32(0xffffffff >>
1788                                                          (32 - width));
1789                 }
1790                 break;
1791         case RTE_FLOW_FIELD_MARK:
1792                 {
1793                         int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
1794                                                        0, error);
1795                         if (reg < 0)
1796                                 return;
1797                         MLX5_ASSERT(reg != REG_NON);
1798                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1799                         info[idx] = (struct field_modify_info){4, 0,
1800                                                 reg_to_field[reg]};
1801                         if (mask)
1802                                 mask[idx] =
1803                                         rte_cpu_to_be_32(0xffffffff >>
1804                                                          (32 - width));
1805                 }
1806                 break;
1807         case RTE_FLOW_FIELD_META:
1808                 {
1809                         unsigned int xmeta = config->dv_xmeta_en;
1810                         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1811                         if (reg < 0)
1812                                 return;
1813                         MLX5_ASSERT(reg != REG_NON);
1814                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1815                         if (xmeta == MLX5_XMETA_MODE_META16) {
1816                                 info[idx] = (struct field_modify_info){2, 0,
1817                                                         reg_to_field[reg]};
1818                                 if (mask)
1819                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1820                                                                 (16 - width));
1821                         } else if (xmeta == MLX5_XMETA_MODE_META32) {
1822                                 info[idx] = (struct field_modify_info){4, 0,
1823                                                         reg_to_field[reg]};
1824                                 if (mask)
1825                                         mask[idx] =
1826                                                 rte_cpu_to_be_32(0xffffffff >>
1827                                                                 (32 - width));
1828                         } else {
1829                                 MLX5_ASSERT(false);
1830                         }
1831                 }
1832                 break;
1833         case RTE_FLOW_FIELD_POINTER:
1834         case RTE_FLOW_FIELD_VALUE:
1835                 if (data->field == RTE_FLOW_FIELD_POINTER)
1836                         memcpy(&val, (void *)(uintptr_t)data->value,
1837                                sizeof(uint64_t));
1838                 else
1839                         val = data->value;
1840                 for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) {
1841                         if (mask[idx]) {
1842                                 if (dst_width == 48) {
1843                                         /*special case for MAC addresses */
1844                                         value[idx] = rte_cpu_to_be_16(val);
1845                                         val >>= 16;
1846                                         dst_width -= 16;
1847                                 } else if (dst_width > 16) {
1848                                         value[idx] = rte_cpu_to_be_32(val);
1849                                         val >>= 32;
1850                                 } else if (dst_width > 8) {
1851                                         value[idx] = rte_cpu_to_be_16(val);
1852                                         val >>= 16;
1853                                 } else {
1854                                         value[idx] = (uint8_t)val;
1855                                         val >>= 8;
1856                                 }
1857                                 if (!val)
1858                                         break;
1859                         }
1860                 }
1861                 break;
1862         default:
1863                 MLX5_ASSERT(false);
1864                 break;
1865         }
1866 }
1867
1868 /**
1869  * Convert modify_field action to DV specification.
1870  *
1871  * @param[in] dev
1872  *   Pointer to the rte_eth_dev structure.
1873  * @param[in,out] resource
1874  *   Pointer to the modify-header resource.
1875  * @param[in] action
1876  *   Pointer to action specification.
1877  * @param[in] attr
1878  *   Attributes of flow that includes this item.
1879  * @param[out] error
1880  *   Pointer to the error structure.
1881  *
1882  * @return
1883  *   0 on success, a negative errno value otherwise and rte_errno is set.
1884  */
1885 static int
1886 flow_dv_convert_action_modify_field
1887                         (struct rte_eth_dev *dev,
1888                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1889                          const struct rte_flow_action *action,
1890                          const struct rte_flow_attr *attr,
1891                          struct rte_flow_error *error)
1892 {
1893         struct mlx5_priv *priv = dev->data->dev_private;
1894         struct mlx5_dev_config *config = &priv->config;
1895         const struct rte_flow_action_modify_field *conf =
1896                 (const struct rte_flow_action_modify_field *)(action->conf);
1897         struct rte_flow_item item;
1898         struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
1899                                                                 {0, 0, 0} };
1900         struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
1901                                                                 {0, 0, 0} };
1902         uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1903         uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1904         uint32_t type;
1905         uint32_t dst_width = mlx5_flow_item_field_width(config,
1906                                                         conf->dst.field);
1907
1908         if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
1909                 conf->src.field == RTE_FLOW_FIELD_VALUE) {
1910                 type = MLX5_MODIFICATION_TYPE_SET;
1911                 /** For SET fill the destination field (field) first. */
1912                 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
1913                         value, conf->width, dst_width, dev, attr, error);
1914                 /** Then copy immediate value from source as per mask. */
1915                 mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask,
1916                         value, conf->width, dst_width, dev, attr, error);
1917                 item.spec = &value;
1918         } else {
1919                 type = MLX5_MODIFICATION_TYPE_COPY;
1920                 /** For COPY fill the destination field (dcopy) without mask. */
1921                 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
1922                         value, conf->width, dst_width, dev, attr, error);
1923                 /** Then construct the source field (field) with mask. */
1924                 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
1925                         value, conf->width, dst_width, dev, attr, error);
1926         }
1927         item.mask = &mask;
1928         return flow_dv_convert_modify_action(&item,
1929                         field, dcopy, resource, type, error);
1930 }
1931
1932 /**
1933  * Validate MARK item.
1934  *
1935  * @param[in] dev
1936  *   Pointer to the rte_eth_dev structure.
1937  * @param[in] item
1938  *   Item specification.
1939  * @param[in] attr
1940  *   Attributes of flow that includes this item.
1941  * @param[out] error
1942  *   Pointer to error structure.
1943  *
1944  * @return
1945  *   0 on success, a negative errno value otherwise and rte_errno is set.
1946  */
1947 static int
1948 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1949                            const struct rte_flow_item *item,
1950                            const struct rte_flow_attr *attr __rte_unused,
1951                            struct rte_flow_error *error)
1952 {
1953         struct mlx5_priv *priv = dev->data->dev_private;
1954         struct mlx5_dev_config *config = &priv->config;
1955         const struct rte_flow_item_mark *spec = item->spec;
1956         const struct rte_flow_item_mark *mask = item->mask;
1957         const struct rte_flow_item_mark nic_mask = {
1958                 .id = priv->sh->dv_mark_mask,
1959         };
1960         int ret;
1961
1962         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1963                 return rte_flow_error_set(error, ENOTSUP,
1964                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1965                                           "extended metadata feature"
1966                                           " isn't enabled");
1967         if (!mlx5_flow_ext_mreg_supported(dev))
1968                 return rte_flow_error_set(error, ENOTSUP,
1969                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1970                                           "extended metadata register"
1971                                           " isn't supported");
1972         if (!nic_mask.id)
1973                 return rte_flow_error_set(error, ENOTSUP,
1974                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1975                                           "extended metadata register"
1976                                           " isn't available");
1977         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1978         if (ret < 0)
1979                 return ret;
1980         if (!spec)
1981                 return rte_flow_error_set(error, EINVAL,
1982                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1983                                           item->spec,
1984                                           "data cannot be empty");
1985         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1986                 return rte_flow_error_set(error, EINVAL,
1987                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1988                                           &spec->id,
1989                                           "mark id exceeds the limit");
1990         if (!mask)
1991                 mask = &nic_mask;
1992         if (!mask->id)
1993                 return rte_flow_error_set(error, EINVAL,
1994                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1995                                         "mask cannot be zero");
1996
1997         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1998                                         (const uint8_t *)&nic_mask,
1999                                         sizeof(struct rte_flow_item_mark),
2000                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2001         if (ret < 0)
2002                 return ret;
2003         return 0;
2004 }
2005
2006 /**
2007  * Validate META item.
2008  *
2009  * @param[in] dev
2010  *   Pointer to the rte_eth_dev structure.
2011  * @param[in] item
2012  *   Item specification.
2013  * @param[in] attr
2014  *   Attributes of flow that includes this item.
2015  * @param[out] error
2016  *   Pointer to error structure.
2017  *
2018  * @return
2019  *   0 on success, a negative errno value otherwise and rte_errno is set.
2020  */
2021 static int
2022 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
2023                            const struct rte_flow_item *item,
2024                            const struct rte_flow_attr *attr,
2025                            struct rte_flow_error *error)
2026 {
2027         struct mlx5_priv *priv = dev->data->dev_private;
2028         struct mlx5_dev_config *config = &priv->config;
2029         const struct rte_flow_item_meta *spec = item->spec;
2030         const struct rte_flow_item_meta *mask = item->mask;
2031         struct rte_flow_item_meta nic_mask = {
2032                 .data = UINT32_MAX
2033         };
2034         int reg;
2035         int ret;
2036
2037         if (!spec)
2038                 return rte_flow_error_set(error, EINVAL,
2039                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2040                                           item->spec,
2041                                           "data cannot be empty");
2042         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2043                 if (!mlx5_flow_ext_mreg_supported(dev))
2044                         return rte_flow_error_set(error, ENOTSUP,
2045                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2046                                           "extended metadata register"
2047                                           " isn't supported");
2048                 reg = flow_dv_get_metadata_reg(dev, attr, error);
2049                 if (reg < 0)
2050                         return reg;
2051                 if (reg == REG_NON)
2052                         return rte_flow_error_set(error, ENOTSUP,
2053                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2054                                         "unavalable extended metadata register");
2055                 if (reg == REG_B)
2056                         return rte_flow_error_set(error, ENOTSUP,
2057                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2058                                           "match on reg_b "
2059                                           "isn't supported");
2060                 if (reg != REG_A)
2061                         nic_mask.data = priv->sh->dv_meta_mask;
2062         } else {
2063                 if (attr->transfer)
2064                         return rte_flow_error_set(error, ENOTSUP,
2065                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2066                                         "extended metadata feature "
2067                                         "should be enabled when "
2068                                         "meta item is requested "
2069                                         "with e-switch mode ");
2070                 if (attr->ingress)
2071                         return rte_flow_error_set(error, ENOTSUP,
2072                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2073                                         "match on metadata for ingress "
2074                                         "is not supported in legacy "
2075                                         "metadata mode");
2076         }
2077         if (!mask)
2078                 mask = &rte_flow_item_meta_mask;
2079         if (!mask->data)
2080                 return rte_flow_error_set(error, EINVAL,
2081                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2082                                         "mask cannot be zero");
2083
2084         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2085                                         (const uint8_t *)&nic_mask,
2086                                         sizeof(struct rte_flow_item_meta),
2087                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2088         return ret;
2089 }
2090
2091 /**
2092  * Validate TAG item.
2093  *
2094  * @param[in] dev
2095  *   Pointer to the rte_eth_dev structure.
2096  * @param[in] item
2097  *   Item specification.
2098  * @param[in] attr
2099  *   Attributes of flow that includes this item.
2100  * @param[out] error
2101  *   Pointer to error structure.
2102  *
2103  * @return
2104  *   0 on success, a negative errno value otherwise and rte_errno is set.
2105  */
2106 static int
2107 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
2108                           const struct rte_flow_item *item,
2109                           const struct rte_flow_attr *attr __rte_unused,
2110                           struct rte_flow_error *error)
2111 {
2112         const struct rte_flow_item_tag *spec = item->spec;
2113         const struct rte_flow_item_tag *mask = item->mask;
2114         const struct rte_flow_item_tag nic_mask = {
2115                 .data = RTE_BE32(UINT32_MAX),
2116                 .index = 0xff,
2117         };
2118         int ret;
2119
2120         if (!mlx5_flow_ext_mreg_supported(dev))
2121                 return rte_flow_error_set(error, ENOTSUP,
2122                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2123                                           "extensive metadata register"
2124                                           " isn't supported");
2125         if (!spec)
2126                 return rte_flow_error_set(error, EINVAL,
2127                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2128                                           item->spec,
2129                                           "data cannot be empty");
2130         if (!mask)
2131                 mask = &rte_flow_item_tag_mask;
2132         if (!mask->data)
2133                 return rte_flow_error_set(error, EINVAL,
2134                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2135                                         "mask cannot be zero");
2136
2137         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2138                                         (const uint8_t *)&nic_mask,
2139                                         sizeof(struct rte_flow_item_tag),
2140                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2141         if (ret < 0)
2142                 return ret;
2143         if (mask->index != 0xff)
2144                 return rte_flow_error_set(error, EINVAL,
2145                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2146                                           "partial mask for tag index"
2147                                           " is not supported");
2148         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
2149         if (ret < 0)
2150                 return ret;
2151         MLX5_ASSERT(ret != REG_NON);
2152         return 0;
2153 }
2154
2155 /**
2156  * Validate vport item.
2157  *
2158  * @param[in] dev
2159  *   Pointer to the rte_eth_dev structure.
2160  * @param[in] item
2161  *   Item specification.
2162  * @param[in] attr
2163  *   Attributes of flow that includes this item.
2164  * @param[in] item_flags
2165  *   Bit-fields that holds the items detected until now.
2166  * @param[out] error
2167  *   Pointer to error structure.
2168  *
2169  * @return
2170  *   0 on success, a negative errno value otherwise and rte_errno is set.
2171  */
2172 static int
2173 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
2174                               const struct rte_flow_item *item,
2175                               const struct rte_flow_attr *attr,
2176                               uint64_t item_flags,
2177                               struct rte_flow_error *error)
2178 {
2179         const struct rte_flow_item_port_id *spec = item->spec;
2180         const struct rte_flow_item_port_id *mask = item->mask;
2181         const struct rte_flow_item_port_id switch_mask = {
2182                         .id = 0xffffffff,
2183         };
2184         struct mlx5_priv *esw_priv;
2185         struct mlx5_priv *dev_priv;
2186         int ret;
2187
2188         if (!attr->transfer)
2189                 return rte_flow_error_set(error, EINVAL,
2190                                           RTE_FLOW_ERROR_TYPE_ITEM,
2191                                           NULL,
2192                                           "match on port id is valid only"
2193                                           " when transfer flag is enabled");
2194         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
2195                 return rte_flow_error_set(error, ENOTSUP,
2196                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2197                                           "multiple source ports are not"
2198                                           " supported");
2199         if (!mask)
2200                 mask = &switch_mask;
2201         if (mask->id != 0xffffffff)
2202                 return rte_flow_error_set(error, ENOTSUP,
2203                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2204                                            mask,
2205                                            "no support for partial mask on"
2206                                            " \"id\" field");
2207         ret = mlx5_flow_item_acceptable
2208                                 (item, (const uint8_t *)mask,
2209                                  (const uint8_t *)&rte_flow_item_port_id_mask,
2210                                  sizeof(struct rte_flow_item_port_id),
2211                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2212         if (ret)
2213                 return ret;
2214         if (!spec)
2215                 return 0;
2216         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
2217         if (!esw_priv)
2218                 return rte_flow_error_set(error, rte_errno,
2219                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2220                                           "failed to obtain E-Switch info for"
2221                                           " port");
2222         dev_priv = mlx5_dev_to_eswitch_info(dev);
2223         if (!dev_priv)
2224                 return rte_flow_error_set(error, rte_errno,
2225                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2226                                           NULL,
2227                                           "failed to obtain E-Switch info");
2228         if (esw_priv->domain_id != dev_priv->domain_id)
2229                 return rte_flow_error_set(error, EINVAL,
2230                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2231                                           "cannot match on a port from a"
2232                                           " different E-Switch");
2233         return 0;
2234 }
2235
2236 /**
2237  * Validate VLAN item.
2238  *
2239  * @param[in] item
2240  *   Item specification.
2241  * @param[in] item_flags
2242  *   Bit-fields that holds the items detected until now.
2243  * @param[in] dev
2244  *   Ethernet device flow is being created on.
2245  * @param[out] error
2246  *   Pointer to error structure.
2247  *
2248  * @return
2249  *   0 on success, a negative errno value otherwise and rte_errno is set.
2250  */
2251 static int
2252 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
2253                            uint64_t item_flags,
2254                            struct rte_eth_dev *dev,
2255                            struct rte_flow_error *error)
2256 {
2257         const struct rte_flow_item_vlan *mask = item->mask;
2258         const struct rte_flow_item_vlan nic_mask = {
2259                 .tci = RTE_BE16(UINT16_MAX),
2260                 .inner_type = RTE_BE16(UINT16_MAX),
2261                 .has_more_vlan = 1,
2262         };
2263         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2264         int ret;
2265         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2266                                         MLX5_FLOW_LAYER_INNER_L4) :
2267                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2268                                         MLX5_FLOW_LAYER_OUTER_L4);
2269         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2270                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2271
2272         if (item_flags & vlanm)
2273                 return rte_flow_error_set(error, EINVAL,
2274                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2275                                           "multiple VLAN layers not supported");
2276         else if ((item_flags & l34m) != 0)
2277                 return rte_flow_error_set(error, EINVAL,
2278                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2279                                           "VLAN cannot follow L3/L4 layer");
2280         if (!mask)
2281                 mask = &rte_flow_item_vlan_mask;
2282         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2283                                         (const uint8_t *)&nic_mask,
2284                                         sizeof(struct rte_flow_item_vlan),
2285                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2286         if (ret)
2287                 return ret;
2288         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2289                 struct mlx5_priv *priv = dev->data->dev_private;
2290
2291                 if (priv->vmwa_context) {
2292                         /*
2293                          * Non-NULL context means we have a virtual machine
2294                          * and SR-IOV enabled, we have to create VLAN interface
2295                          * to make hypervisor to setup E-Switch vport
2296                          * context correctly. We avoid creating the multiple
2297                          * VLAN interfaces, so we cannot support VLAN tag mask.
2298                          */
2299                         return rte_flow_error_set(error, EINVAL,
2300                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2301                                                   item,
2302                                                   "VLAN tag mask is not"
2303                                                   " supported in virtual"
2304                                                   " environment");
2305                 }
2306         }
2307         return 0;
2308 }
2309
2310 /*
2311  * GTP flags are contained in 1 byte of the format:
2312  * -------------------------------------------
2313  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
2314  * |-----------------------------------------|
2315  * | value | Version | PT | Res | E | S | PN |
2316  * -------------------------------------------
2317  *
2318  * Matching is supported only for GTP flags E, S, PN.
2319  */
2320 #define MLX5_GTP_FLAGS_MASK     0x07
2321
2322 /**
2323  * Validate GTP item.
2324  *
2325  * @param[in] dev
2326  *   Pointer to the rte_eth_dev structure.
2327  * @param[in] item
2328  *   Item specification.
2329  * @param[in] item_flags
2330  *   Bit-fields that holds the items detected until now.
2331  * @param[out] error
2332  *   Pointer to error structure.
2333  *
2334  * @return
2335  *   0 on success, a negative errno value otherwise and rte_errno is set.
2336  */
2337 static int
2338 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
2339                           const struct rte_flow_item *item,
2340                           uint64_t item_flags,
2341                           struct rte_flow_error *error)
2342 {
2343         struct mlx5_priv *priv = dev->data->dev_private;
2344         const struct rte_flow_item_gtp *spec = item->spec;
2345         const struct rte_flow_item_gtp *mask = item->mask;
2346         const struct rte_flow_item_gtp nic_mask = {
2347                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
2348                 .msg_type = 0xff,
2349                 .teid = RTE_BE32(0xffffffff),
2350         };
2351
2352         if (!priv->config.hca_attr.tunnel_stateless_gtp)
2353                 return rte_flow_error_set(error, ENOTSUP,
2354                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2355                                           "GTP support is not enabled");
2356         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2357                 return rte_flow_error_set(error, ENOTSUP,
2358                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2359                                           "multiple tunnel layers not"
2360                                           " supported");
2361         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2362                 return rte_flow_error_set(error, EINVAL,
2363                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2364                                           "no outer UDP layer found");
2365         if (!mask)
2366                 mask = &rte_flow_item_gtp_mask;
2367         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
2368                 return rte_flow_error_set(error, ENOTSUP,
2369                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2370                                           "Match is supported for GTP"
2371                                           " flags only");
2372         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2373                                          (const uint8_t *)&nic_mask,
2374                                          sizeof(struct rte_flow_item_gtp),
2375                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2376 }
2377
2378 /**
2379  * Validate GTP PSC item.
2380  *
2381  * @param[in] item
2382  *   Item specification.
2383  * @param[in] last_item
2384  *   Previous validated item in the pattern items.
2385  * @param[in] gtp_item
2386  *   Previous GTP item specification.
2387  * @param[in] attr
2388  *   Pointer to flow attributes.
2389  * @param[out] error
2390  *   Pointer to error structure.
2391  *
2392  * @return
2393  *   0 on success, a negative errno value otherwise and rte_errno is set.
2394  */
2395 static int
2396 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
2397                               uint64_t last_item,
2398                               const struct rte_flow_item *gtp_item,
2399                               const struct rte_flow_attr *attr,
2400                               struct rte_flow_error *error)
2401 {
2402         const struct rte_flow_item_gtp *gtp_spec;
2403         const struct rte_flow_item_gtp *gtp_mask;
2404         const struct rte_flow_item_gtp_psc *spec;
2405         const struct rte_flow_item_gtp_psc *mask;
2406         const struct rte_flow_item_gtp_psc nic_mask = {
2407                 .pdu_type = 0xFF,
2408                 .qfi = 0xFF,
2409         };
2410
2411         if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
2412                 return rte_flow_error_set
2413                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2414                          "GTP PSC item must be preceded with GTP item");
2415         gtp_spec = gtp_item->spec;
2416         gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
2417         /* GTP spec and E flag is requested to match zero. */
2418         if (gtp_spec &&
2419                 (gtp_mask->v_pt_rsv_flags &
2420                 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
2421                 return rte_flow_error_set
2422                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2423                          "GTP E flag must be 1 to match GTP PSC");
2424         /* Check the flow is not created in group zero. */
2425         if (!attr->transfer && !attr->group)
2426                 return rte_flow_error_set
2427                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2428                          "GTP PSC is not supported for group 0");
2429         /* GTP spec is here and E flag is requested to match zero. */
2430         if (!item->spec)
2431                 return 0;
2432         spec = item->spec;
2433         mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
2434         if (spec->pdu_type > MLX5_GTP_EXT_MAX_PDU_TYPE)
2435                 return rte_flow_error_set
2436                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2437                          "PDU type should be smaller than 16");
2438         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2439                                          (const uint8_t *)&nic_mask,
2440                                          sizeof(struct rte_flow_item_gtp_psc),
2441                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2442 }
2443
2444 /**
2445  * Validate IPV4 item.
2446  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
2447  * add specific validation of fragment_offset field,
2448  *
2449  * @param[in] item
2450  *   Item specification.
2451  * @param[in] item_flags
2452  *   Bit-fields that holds the items detected until now.
2453  * @param[out] error
2454  *   Pointer to error structure.
2455  *
2456  * @return
2457  *   0 on success, a negative errno value otherwise and rte_errno is set.
2458  */
2459 static int
2460 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
2461                            uint64_t item_flags,
2462                            uint64_t last_item,
2463                            uint16_t ether_type,
2464                            struct rte_flow_error *error)
2465 {
2466         int ret;
2467         const struct rte_flow_item_ipv4 *spec = item->spec;
2468         const struct rte_flow_item_ipv4 *last = item->last;
2469         const struct rte_flow_item_ipv4 *mask = item->mask;
2470         rte_be16_t fragment_offset_spec = 0;
2471         rte_be16_t fragment_offset_last = 0;
2472         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
2473                 .hdr = {
2474                         .src_addr = RTE_BE32(0xffffffff),
2475                         .dst_addr = RTE_BE32(0xffffffff),
2476                         .type_of_service = 0xff,
2477                         .fragment_offset = RTE_BE16(0xffff),
2478                         .next_proto_id = 0xff,
2479                         .time_to_live = 0xff,
2480                 },
2481         };
2482
2483         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
2484                                            ether_type, &nic_ipv4_mask,
2485                                            MLX5_ITEM_RANGE_ACCEPTED, error);
2486         if (ret < 0)
2487                 return ret;
2488         if (spec && mask)
2489                 fragment_offset_spec = spec->hdr.fragment_offset &
2490                                        mask->hdr.fragment_offset;
2491         if (!fragment_offset_spec)
2492                 return 0;
2493         /*
2494          * spec and mask are valid, enforce using full mask to make sure the
2495          * complete value is used correctly.
2496          */
2497         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2498                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2499                 return rte_flow_error_set(error, EINVAL,
2500                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2501                                           item, "must use full mask for"
2502                                           " fragment_offset");
2503         /*
2504          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
2505          * indicating this is 1st fragment of fragmented packet.
2506          * This is not yet supported in MLX5, return appropriate error message.
2507          */
2508         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
2509                 return rte_flow_error_set(error, ENOTSUP,
2510                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2511                                           "match on first fragment not "
2512                                           "supported");
2513         if (fragment_offset_spec && !last)
2514                 return rte_flow_error_set(error, ENOTSUP,
2515                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2516                                           "specified value not supported");
2517         /* spec and last are valid, validate the specified range. */
2518         fragment_offset_last = last->hdr.fragment_offset &
2519                                mask->hdr.fragment_offset;
2520         /*
2521          * Match on fragment_offset spec 0x2001 and last 0x3fff
2522          * means MF is 1 and frag-offset is > 0.
2523          * This packet is fragment 2nd and onward, excluding last.
2524          * This is not yet supported in MLX5, return appropriate
2525          * error message.
2526          */
2527         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
2528             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2529                 return rte_flow_error_set(error, ENOTSUP,
2530                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2531                                           last, "match on following "
2532                                           "fragments not supported");
2533         /*
2534          * Match on fragment_offset spec 0x0001 and last 0x1fff
2535          * means MF is 0 and frag-offset is > 0.
2536          * This packet is last fragment of fragmented packet.
2537          * This is not yet supported in MLX5, return appropriate
2538          * error message.
2539          */
2540         if (fragment_offset_spec == RTE_BE16(1) &&
2541             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
2542                 return rte_flow_error_set(error, ENOTSUP,
2543                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2544                                           last, "match on last "
2545                                           "fragment not supported");
2546         /*
2547          * Match on fragment_offset spec 0x0001 and last 0x3fff
2548          * means MF and/or frag-offset is not 0.
2549          * This is a fragmented packet.
2550          * Other range values are invalid and rejected.
2551          */
2552         if (!(fragment_offset_spec == RTE_BE16(1) &&
2553               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
2554                 return rte_flow_error_set(error, ENOTSUP,
2555                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2556                                           "specified range not supported");
2557         return 0;
2558 }
2559
2560 /**
2561  * Validate IPV6 fragment extension item.
2562  *
2563  * @param[in] item
2564  *   Item specification.
2565  * @param[in] item_flags
2566  *   Bit-fields that holds the items detected until now.
2567  * @param[out] error
2568  *   Pointer to error structure.
2569  *
2570  * @return
2571  *   0 on success, a negative errno value otherwise and rte_errno is set.
2572  */
2573 static int
2574 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
2575                                     uint64_t item_flags,
2576                                     struct rte_flow_error *error)
2577 {
2578         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
2579         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
2580         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
2581         rte_be16_t frag_data_spec = 0;
2582         rte_be16_t frag_data_last = 0;
2583         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2584         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2585                                       MLX5_FLOW_LAYER_OUTER_L4;
2586         int ret = 0;
2587         struct rte_flow_item_ipv6_frag_ext nic_mask = {
2588                 .hdr = {
2589                         .next_header = 0xff,
2590                         .frag_data = RTE_BE16(0xffff),
2591                 },
2592         };
2593
2594         if (item_flags & l4m)
2595                 return rte_flow_error_set(error, EINVAL,
2596                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2597                                           "ipv6 fragment extension item cannot "
2598                                           "follow L4 item.");
2599         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
2600             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2601                 return rte_flow_error_set(error, EINVAL,
2602                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2603                                           "ipv6 fragment extension item must "
2604                                           "follow ipv6 item");
2605         if (spec && mask)
2606                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
2607         if (!frag_data_spec)
2608                 return 0;
2609         /*
2610          * spec and mask are valid, enforce using full mask to make sure the
2611          * complete value is used correctly.
2612          */
2613         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
2614                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2615                 return rte_flow_error_set(error, EINVAL,
2616                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2617                                           item, "must use full mask for"
2618                                           " frag_data");
2619         /*
2620          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
2621          * This is 1st fragment of fragmented packet.
2622          */
2623         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
2624                 return rte_flow_error_set(error, ENOTSUP,
2625                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2626                                           "match on first fragment not "
2627                                           "supported");
2628         if (frag_data_spec && !last)
2629                 return rte_flow_error_set(error, EINVAL,
2630                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2631                                           "specified value not supported");
2632         ret = mlx5_flow_item_acceptable
2633                                 (item, (const uint8_t *)mask,
2634                                  (const uint8_t *)&nic_mask,
2635                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
2636                                  MLX5_ITEM_RANGE_ACCEPTED, error);
2637         if (ret)
2638                 return ret;
2639         /* spec and last are valid, validate the specified range. */
2640         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
2641         /*
2642          * Match on frag_data spec 0x0009 and last 0xfff9
2643          * means M is 1 and frag-offset is > 0.
2644          * This packet is fragment 2nd and onward, excluding last.
2645          * This is not yet supported in MLX5, return appropriate
2646          * error message.
2647          */
2648         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2649                                        RTE_IPV6_EHDR_MF_MASK) &&
2650             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2651                 return rte_flow_error_set(error, ENOTSUP,
2652                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2653                                           last, "match on following "
2654                                           "fragments not supported");
2655         /*
2656          * Match on frag_data spec 0x0008 and last 0xfff8
2657          * means M is 0 and frag-offset is > 0.
2658          * This packet is last fragment of fragmented packet.
2659          * This is not yet supported in MLX5, return appropriate
2660          * error message.
2661          */
2662         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2663             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2664                 return rte_flow_error_set(error, ENOTSUP,
2665                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2666                                           last, "match on last "
2667                                           "fragment not supported");
2668         /* Other range values are invalid and rejected. */
2669         return rte_flow_error_set(error, EINVAL,
2670                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2671                                   "specified range not supported");
2672 }
2673
2674 /*
2675  * Validate ASO CT item.
2676  *
2677  * @param[in] dev
2678  *   Pointer to the rte_eth_dev structure.
2679  * @param[in] item
2680  *   Item specification.
2681  * @param[in] item_flags
2682  *   Pointer to bit-fields that holds the items detected until now.
2683  * @param[out] error
2684  *   Pointer to error structure.
2685  *
2686  * @return
2687  *   0 on success, a negative errno value otherwise and rte_errno is set.
2688  */
2689 static int
2690 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
2691                              const struct rte_flow_item *item,
2692                              uint64_t *item_flags,
2693                              struct rte_flow_error *error)
2694 {
2695         const struct rte_flow_item_conntrack *spec = item->spec;
2696         const struct rte_flow_item_conntrack *mask = item->mask;
2697         RTE_SET_USED(dev);
2698         uint32_t flags;
2699
2700         if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
2701                 return rte_flow_error_set(error, EINVAL,
2702                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2703                                           "Only one CT is supported");
2704         if (!mask)
2705                 mask = &rte_flow_item_conntrack_mask;
2706         flags = spec->flags & mask->flags;
2707         if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
2708             ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
2709              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
2710              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
2711                 return rte_flow_error_set(error, EINVAL,
2712                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2713                                           "Conflict status bits");
2714         /* State change also needs to be considered. */
2715         *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
2716         return 0;
2717 }
2718
2719 /**
2720  * Validate the pop VLAN action.
2721  *
2722  * @param[in] dev
2723  *   Pointer to the rte_eth_dev structure.
2724  * @param[in] action_flags
2725  *   Holds the actions detected until now.
2726  * @param[in] action
2727  *   Pointer to the pop vlan action.
2728  * @param[in] item_flags
2729  *   The items found in this flow rule.
2730  * @param[in] attr
2731  *   Pointer to flow attributes.
2732  * @param[out] error
2733  *   Pointer to error structure.
2734  *
2735  * @return
2736  *   0 on success, a negative errno value otherwise and rte_errno is set.
2737  */
2738 static int
2739 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2740                                  uint64_t action_flags,
2741                                  const struct rte_flow_action *action,
2742                                  uint64_t item_flags,
2743                                  const struct rte_flow_attr *attr,
2744                                  struct rte_flow_error *error)
2745 {
2746         const struct mlx5_priv *priv = dev->data->dev_private;
2747
2748         (void)action;
2749         (void)attr;
2750         if (!priv->sh->pop_vlan_action)
2751                 return rte_flow_error_set(error, ENOTSUP,
2752                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2753                                           NULL,
2754                                           "pop vlan action is not supported");
2755         if (attr->egress)
2756                 return rte_flow_error_set(error, ENOTSUP,
2757                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2758                                           NULL,
2759                                           "pop vlan action not supported for "
2760                                           "egress");
2761         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2762                 return rte_flow_error_set(error, ENOTSUP,
2763                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2764                                           "no support for multiple VLAN "
2765                                           "actions");
2766         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2767         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2768             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2769                 return rte_flow_error_set(error, ENOTSUP,
2770                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2771                                           NULL,
2772                                           "cannot pop vlan after decap without "
2773                                           "match on inner vlan in the flow");
2774         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2775         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2776             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2777                 return rte_flow_error_set(error, ENOTSUP,
2778                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2779                                           NULL,
2780                                           "cannot pop vlan without a "
2781                                           "match on (outer) vlan in the flow");
2782         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2783                 return rte_flow_error_set(error, EINVAL,
2784                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2785                                           "wrong action order, port_id should "
2786                                           "be after pop VLAN action");
2787         if (!attr->transfer && priv->representor)
2788                 return rte_flow_error_set(error, ENOTSUP,
2789                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2790                                           "pop vlan action for VF representor "
2791                                           "not supported on NIC table");
2792         return 0;
2793 }
2794
2795 /**
2796  * Get VLAN default info from vlan match info.
2797  *
2798  * @param[in] items
2799  *   the list of item specifications.
2800  * @param[out] vlan
2801  *   pointer VLAN info to fill to.
2802  *
2803  * @return
2804  *   0 on success, a negative errno value otherwise and rte_errno is set.
2805  */
2806 static void
2807 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2808                                   struct rte_vlan_hdr *vlan)
2809 {
2810         const struct rte_flow_item_vlan nic_mask = {
2811                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2812                                 MLX5DV_FLOW_VLAN_VID_MASK),
2813                 .inner_type = RTE_BE16(0xffff),
2814         };
2815
2816         if (items == NULL)
2817                 return;
2818         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2819                 int type = items->type;
2820
2821                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2822                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2823                         break;
2824         }
2825         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2826                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2827                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2828
2829                 /* If VLAN item in pattern doesn't contain data, return here. */
2830                 if (!vlan_v)
2831                         return;
2832                 if (!vlan_m)
2833                         vlan_m = &nic_mask;
2834                 /* Only full match values are accepted */
2835                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2836                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2837                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2838                         vlan->vlan_tci |=
2839                                 rte_be_to_cpu_16(vlan_v->tci &
2840                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2841                 }
2842                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2843                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2844                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2845                         vlan->vlan_tci |=
2846                                 rte_be_to_cpu_16(vlan_v->tci &
2847                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2848                 }
2849                 if (vlan_m->inner_type == nic_mask.inner_type)
2850                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2851                                                            vlan_m->inner_type);
2852         }
2853 }
2854
2855 /**
2856  * Validate the push VLAN action.
2857  *
2858  * @param[in] dev
2859  *   Pointer to the rte_eth_dev structure.
2860  * @param[in] action_flags
2861  *   Holds the actions detected until now.
2862  * @param[in] item_flags
2863  *   The items found in this flow rule.
2864  * @param[in] action
2865  *   Pointer to the action structure.
2866  * @param[in] attr
2867  *   Pointer to flow attributes
2868  * @param[out] error
2869  *   Pointer to error structure.
2870  *
2871  * @return
2872  *   0 on success, a negative errno value otherwise and rte_errno is set.
2873  */
2874 static int
2875 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2876                                   uint64_t action_flags,
2877                                   const struct rte_flow_item_vlan *vlan_m,
2878                                   const struct rte_flow_action *action,
2879                                   const struct rte_flow_attr *attr,
2880                                   struct rte_flow_error *error)
2881 {
2882         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2883         const struct mlx5_priv *priv = dev->data->dev_private;
2884
2885         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2886             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2887                 return rte_flow_error_set(error, EINVAL,
2888                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2889                                           "invalid vlan ethertype");
2890         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2891                 return rte_flow_error_set(error, EINVAL,
2892                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2893                                           "wrong action order, port_id should "
2894                                           "be after push VLAN");
2895         if (!attr->transfer && priv->representor)
2896                 return rte_flow_error_set(error, ENOTSUP,
2897                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2898                                           "push vlan action for VF representor "
2899                                           "not supported on NIC table");
2900         if (vlan_m &&
2901             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2902             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2903                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2904             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2905             !(mlx5_flow_find_action
2906                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2907                 return rte_flow_error_set(error, EINVAL,
2908                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2909                                           "not full match mask on VLAN PCP and "
2910                                           "there is no of_set_vlan_pcp action, "
2911                                           "push VLAN action cannot figure out "
2912                                           "PCP value");
2913         if (vlan_m &&
2914             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2915             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2916                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2917             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2918             !(mlx5_flow_find_action
2919                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2920                 return rte_flow_error_set(error, EINVAL,
2921                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2922                                           "not full match mask on VLAN VID and "
2923                                           "there is no of_set_vlan_vid action, "
2924                                           "push VLAN action cannot figure out "
2925                                           "VID value");
2926         (void)attr;
2927         return 0;
2928 }
2929
2930 /**
2931  * Validate the set VLAN PCP.
2932  *
2933  * @param[in] action_flags
2934  *   Holds the actions detected until now.
2935  * @param[in] actions
2936  *   Pointer to the list of actions remaining in the flow rule.
2937  * @param[out] error
2938  *   Pointer to error structure.
2939  *
2940  * @return
2941  *   0 on success, a negative errno value otherwise and rte_errno is set.
2942  */
2943 static int
2944 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2945                                      const struct rte_flow_action actions[],
2946                                      struct rte_flow_error *error)
2947 {
2948         const struct rte_flow_action *action = actions;
2949         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2950
2951         if (conf->vlan_pcp > 7)
2952                 return rte_flow_error_set(error, EINVAL,
2953                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2954                                           "VLAN PCP value is too big");
2955         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2956                 return rte_flow_error_set(error, ENOTSUP,
2957                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2958                                           "set VLAN PCP action must follow "
2959                                           "the push VLAN action");
2960         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2961                 return rte_flow_error_set(error, ENOTSUP,
2962                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2963                                           "Multiple VLAN PCP modification are "
2964                                           "not supported");
2965         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2966                 return rte_flow_error_set(error, EINVAL,
2967                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2968                                           "wrong action order, port_id should "
2969                                           "be after set VLAN PCP");
2970         return 0;
2971 }
2972
2973 /**
2974  * Validate the set VLAN VID.
2975  *
2976  * @param[in] item_flags
2977  *   Holds the items detected in this rule.
2978  * @param[in] action_flags
2979  *   Holds the actions detected until now.
2980  * @param[in] actions
2981  *   Pointer to the list of actions remaining in the flow rule.
2982  * @param[out] error
2983  *   Pointer to error structure.
2984  *
2985  * @return
2986  *   0 on success, a negative errno value otherwise and rte_errno is set.
2987  */
2988 static int
2989 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2990                                      uint64_t action_flags,
2991                                      const struct rte_flow_action actions[],
2992                                      struct rte_flow_error *error)
2993 {
2994         const struct rte_flow_action *action = actions;
2995         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2996
2997         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2998                 return rte_flow_error_set(error, EINVAL,
2999                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3000                                           "VLAN VID value is too big");
3001         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
3002             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
3003                 return rte_flow_error_set(error, ENOTSUP,
3004                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3005                                           "set VLAN VID action must follow push"
3006                                           " VLAN action or match on VLAN item");
3007         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
3008                 return rte_flow_error_set(error, ENOTSUP,
3009                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3010                                           "Multiple VLAN VID modifications are "
3011                                           "not supported");
3012         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3013                 return rte_flow_error_set(error, EINVAL,
3014                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3015                                           "wrong action order, port_id should "
3016                                           "be after set VLAN VID");
3017         return 0;
3018 }
3019
3020 /*
3021  * Validate the FLAG action.
3022  *
3023  * @param[in] dev
3024  *   Pointer to the rte_eth_dev structure.
3025  * @param[in] action_flags
3026  *   Holds the actions detected until now.
3027  * @param[in] attr
3028  *   Pointer to flow attributes
3029  * @param[out] error
3030  *   Pointer to error structure.
3031  *
3032  * @return
3033  *   0 on success, a negative errno value otherwise and rte_errno is set.
3034  */
3035 static int
3036 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
3037                              uint64_t action_flags,
3038                              const struct rte_flow_attr *attr,
3039                              struct rte_flow_error *error)
3040 {
3041         struct mlx5_priv *priv = dev->data->dev_private;
3042         struct mlx5_dev_config *config = &priv->config;
3043         int ret;
3044
3045         /* Fall back if no extended metadata register support. */
3046         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3047                 return mlx5_flow_validate_action_flag(action_flags, attr,
3048                                                       error);
3049         /* Extensive metadata mode requires registers. */
3050         if (!mlx5_flow_ext_mreg_supported(dev))
3051                 return rte_flow_error_set(error, ENOTSUP,
3052                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3053                                           "no metadata registers "
3054                                           "to support flag action");
3055         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
3056                 return rte_flow_error_set(error, ENOTSUP,
3057                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3058                                           "extended metadata register"
3059                                           " isn't available");
3060         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3061         if (ret < 0)
3062                 return ret;
3063         MLX5_ASSERT(ret > 0);
3064         if (action_flags & MLX5_FLOW_ACTION_MARK)
3065                 return rte_flow_error_set(error, EINVAL,
3066                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3067                                           "can't mark and flag in same flow");
3068         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3069                 return rte_flow_error_set(error, EINVAL,
3070                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3071                                           "can't have 2 flag"
3072                                           " actions in same flow");
3073         return 0;
3074 }
3075
3076 /**
3077  * Validate MARK action.
3078  *
3079  * @param[in] dev
3080  *   Pointer to the rte_eth_dev structure.
3081  * @param[in] action
3082  *   Pointer to action.
3083  * @param[in] action_flags
3084  *   Holds the actions detected until now.
3085  * @param[in] attr
3086  *   Pointer to flow attributes
3087  * @param[out] error
3088  *   Pointer to error structure.
3089  *
3090  * @return
3091  *   0 on success, a negative errno value otherwise and rte_errno is set.
3092  */
3093 static int
3094 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
3095                              const struct rte_flow_action *action,
3096                              uint64_t action_flags,
3097                              const struct rte_flow_attr *attr,
3098                              struct rte_flow_error *error)
3099 {
3100         struct mlx5_priv *priv = dev->data->dev_private;
3101         struct mlx5_dev_config *config = &priv->config;
3102         const struct rte_flow_action_mark *mark = action->conf;
3103         int ret;
3104
3105         if (is_tunnel_offload_active(dev))
3106                 return rte_flow_error_set(error, ENOTSUP,
3107                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3108                                           "no mark action "
3109                                           "if tunnel offload active");
3110         /* Fall back if no extended metadata register support. */
3111         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3112                 return mlx5_flow_validate_action_mark(action, action_flags,
3113                                                       attr, error);
3114         /* Extensive metadata mode requires registers. */
3115         if (!mlx5_flow_ext_mreg_supported(dev))
3116                 return rte_flow_error_set(error, ENOTSUP,
3117                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3118                                           "no metadata registers "
3119                                           "to support mark action");
3120         if (!priv->sh->dv_mark_mask)
3121                 return rte_flow_error_set(error, ENOTSUP,
3122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3123                                           "extended metadata register"
3124                                           " isn't available");
3125         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3126         if (ret < 0)
3127                 return ret;
3128         MLX5_ASSERT(ret > 0);
3129         if (!mark)
3130                 return rte_flow_error_set(error, EINVAL,
3131                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3132                                           "configuration cannot be null");
3133         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
3134                 return rte_flow_error_set(error, EINVAL,
3135                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3136                                           &mark->id,
3137                                           "mark id exceeds the limit");
3138         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3139                 return rte_flow_error_set(error, EINVAL,
3140                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3141                                           "can't flag and mark in same flow");
3142         if (action_flags & MLX5_FLOW_ACTION_MARK)
3143                 return rte_flow_error_set(error, EINVAL,
3144                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3145                                           "can't have 2 mark actions in same"
3146                                           " flow");
3147         return 0;
3148 }
3149
3150 /**
3151  * Validate SET_META action.
3152  *
3153  * @param[in] dev
3154  *   Pointer to the rte_eth_dev structure.
3155  * @param[in] action
3156  *   Pointer to the action structure.
3157  * @param[in] action_flags
3158  *   Holds the actions detected until now.
3159  * @param[in] attr
3160  *   Pointer to flow attributes
3161  * @param[out] error
3162  *   Pointer to error structure.
3163  *
3164  * @return
3165  *   0 on success, a negative errno value otherwise and rte_errno is set.
3166  */
3167 static int
3168 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
3169                                  const struct rte_flow_action *action,
3170                                  uint64_t action_flags __rte_unused,
3171                                  const struct rte_flow_attr *attr,
3172                                  struct rte_flow_error *error)
3173 {
3174         const struct rte_flow_action_set_meta *conf;
3175         uint32_t nic_mask = UINT32_MAX;
3176         int reg;
3177
3178         if (!mlx5_flow_ext_mreg_supported(dev))
3179                 return rte_flow_error_set(error, ENOTSUP,
3180                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3181                                           "extended metadata register"
3182                                           " isn't supported");
3183         reg = flow_dv_get_metadata_reg(dev, attr, error);
3184         if (reg < 0)
3185                 return reg;
3186         if (reg == REG_NON)
3187                 return rte_flow_error_set(error, ENOTSUP,
3188                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3189                                           "unavalable extended metadata register");
3190         if (reg != REG_A && reg != REG_B) {
3191                 struct mlx5_priv *priv = dev->data->dev_private;
3192
3193                 nic_mask = priv->sh->dv_meta_mask;
3194         }
3195         if (!(action->conf))
3196                 return rte_flow_error_set(error, EINVAL,
3197                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3198                                           "configuration cannot be null");
3199         conf = (const struct rte_flow_action_set_meta *)action->conf;
3200         if (!conf->mask)
3201                 return rte_flow_error_set(error, EINVAL,
3202                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3203                                           "zero mask doesn't have any effect");
3204         if (conf->mask & ~nic_mask)
3205                 return rte_flow_error_set(error, EINVAL,
3206                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3207                                           "meta data must be within reg C0");
3208         return 0;
3209 }
3210
3211 /**
3212  * Validate SET_TAG action.
3213  *
3214  * @param[in] dev
3215  *   Pointer to the rte_eth_dev structure.
3216  * @param[in] action
3217  *   Pointer to the action structure.
3218  * @param[in] action_flags
3219  *   Holds the actions detected until now.
3220  * @param[in] attr
3221  *   Pointer to flow attributes
3222  * @param[out] error
3223  *   Pointer to error structure.
3224  *
3225  * @return
3226  *   0 on success, a negative errno value otherwise and rte_errno is set.
3227  */
3228 static int
3229 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
3230                                 const struct rte_flow_action *action,
3231                                 uint64_t action_flags,
3232                                 const struct rte_flow_attr *attr,
3233                                 struct rte_flow_error *error)
3234 {
3235         const struct rte_flow_action_set_tag *conf;
3236         const uint64_t terminal_action_flags =
3237                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
3238                 MLX5_FLOW_ACTION_RSS;
3239         int ret;
3240
3241         if (!mlx5_flow_ext_mreg_supported(dev))
3242                 return rte_flow_error_set(error, ENOTSUP,
3243                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3244                                           "extensive metadata register"
3245                                           " isn't supported");
3246         if (!(action->conf))
3247                 return rte_flow_error_set(error, EINVAL,
3248                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3249                                           "configuration cannot be null");
3250         conf = (const struct rte_flow_action_set_tag *)action->conf;
3251         if (!conf->mask)
3252                 return rte_flow_error_set(error, EINVAL,
3253                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3254                                           "zero mask doesn't have any effect");
3255         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
3256         if (ret < 0)
3257                 return ret;
3258         if (!attr->transfer && attr->ingress &&
3259             (action_flags & terminal_action_flags))
3260                 return rte_flow_error_set(error, EINVAL,
3261                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3262                                           "set_tag has no effect"
3263                                           " with terminal actions");
3264         return 0;
3265 }
3266
3267 /**
3268  * Check if action counter is shared by either old or new mechanism.
3269  *
3270  * @param[in] action
3271  *   Pointer to the action structure.
3272  *
3273  * @return
3274  *   True when counter is shared, false otherwise.
3275  */
3276 static inline bool
3277 is_shared_action_count(const struct rte_flow_action *action)
3278 {
3279         const struct rte_flow_action_count *count =
3280                         (const struct rte_flow_action_count *)action->conf;
3281
3282         if ((int)action->type == MLX5_RTE_FLOW_ACTION_TYPE_COUNT)
3283                 return true;
3284         return !!(count && count->shared);
3285 }
3286
3287 /**
3288  * Validate count action.
3289  *
3290  * @param[in] dev
3291  *   Pointer to rte_eth_dev structure.
3292  * @param[in] shared
3293  *   Indicator if action is shared.
3294  * @param[in] action_flags
3295  *   Holds the actions detected until now.
3296  * @param[out] error
3297  *   Pointer to error structure.
3298  *
3299  * @return
3300  *   0 on success, a negative errno value otherwise and rte_errno is set.
3301  */
3302 static int
3303 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
3304                               uint64_t action_flags,
3305                               struct rte_flow_error *error)
3306 {
3307         struct mlx5_priv *priv = dev->data->dev_private;
3308
3309         if (!priv->config.devx)
3310                 goto notsup_err;
3311         if (action_flags & MLX5_FLOW_ACTION_COUNT)
3312                 return rte_flow_error_set(error, EINVAL,
3313                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3314                                           "duplicate count actions set");
3315         if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
3316             !priv->sh->flow_hit_aso_en)
3317                 return rte_flow_error_set(error, EINVAL,
3318                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3319                                           "old age and shared count combination is not supported");
3320 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
3321         return 0;
3322 #endif
3323 notsup_err:
3324         return rte_flow_error_set
3325                       (error, ENOTSUP,
3326                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3327                        NULL,
3328                        "count action not supported");
3329 }
3330
3331 /**
3332  * Validate the L2 encap action.
3333  *
3334  * @param[in] dev
3335  *   Pointer to the rte_eth_dev structure.
3336  * @param[in] action_flags
3337  *   Holds the actions detected until now.
3338  * @param[in] action
3339  *   Pointer to the action structure.
3340  * @param[in] attr
3341  *   Pointer to flow attributes.
3342  * @param[out] error
3343  *   Pointer to error structure.
3344  *
3345  * @return
3346  *   0 on success, a negative errno value otherwise and rte_errno is set.
3347  */
3348 static int
3349 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
3350                                  uint64_t action_flags,
3351                                  const struct rte_flow_action *action,
3352                                  const struct rte_flow_attr *attr,
3353                                  struct rte_flow_error *error)
3354 {
3355         const struct mlx5_priv *priv = dev->data->dev_private;
3356
3357         if (!(action->conf))
3358                 return rte_flow_error_set(error, EINVAL,
3359                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3360                                           "configuration cannot be null");
3361         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3362                 return rte_flow_error_set(error, EINVAL,
3363                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3364                                           "can only have a single encap action "
3365                                           "in a flow");
3366         if (!attr->transfer && priv->representor)
3367                 return rte_flow_error_set(error, ENOTSUP,
3368                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3369                                           "encap action for VF representor "
3370                                           "not supported on NIC table");
3371         return 0;
3372 }
3373
3374 /**
3375  * Validate a decap action.
3376  *
3377  * @param[in] dev
3378  *   Pointer to the rte_eth_dev structure.
3379  * @param[in] action_flags
3380  *   Holds the actions detected until now.
3381  * @param[in] action
3382  *   Pointer to the action structure.
3383  * @param[in] item_flags
3384  *   Holds the items detected.
3385  * @param[in] attr
3386  *   Pointer to flow attributes
3387  * @param[out] error
3388  *   Pointer to error structure.
3389  *
3390  * @return
3391  *   0 on success, a negative errno value otherwise and rte_errno is set.
3392  */
3393 static int
3394 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
3395                               uint64_t action_flags,
3396                               const struct rte_flow_action *action,
3397                               const uint64_t item_flags,
3398                               const struct rte_flow_attr *attr,
3399                               struct rte_flow_error *error)
3400 {
3401         const struct mlx5_priv *priv = dev->data->dev_private;
3402
3403         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
3404             !priv->config.decap_en)
3405                 return rte_flow_error_set(error, ENOTSUP,
3406                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3407                                           "decap is not enabled");
3408         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
3409                 return rte_flow_error_set(error, ENOTSUP,
3410                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3411                                           action_flags &
3412                                           MLX5_FLOW_ACTION_DECAP ? "can only "
3413                                           "have a single decap action" : "decap "
3414                                           "after encap is not supported");
3415         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
3416                 return rte_flow_error_set(error, EINVAL,
3417                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3418                                           "can't have decap action after"
3419                                           " modify action");
3420         if (attr->egress)
3421                 return rte_flow_error_set(error, ENOTSUP,
3422                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
3423                                           NULL,
3424                                           "decap action not supported for "
3425                                           "egress");
3426         if (!attr->transfer && priv->representor)
3427                 return rte_flow_error_set(error, ENOTSUP,
3428                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3429                                           "decap action for VF representor "
3430                                           "not supported on NIC table");
3431         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
3432             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
3433                 return rte_flow_error_set(error, ENOTSUP,
3434                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3435                                 "VXLAN item should be present for VXLAN decap");
3436         return 0;
3437 }
3438
3439 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
3440
3441 /**
3442  * Validate the raw encap and decap actions.
3443  *
3444  * @param[in] dev
3445  *   Pointer to the rte_eth_dev structure.
3446  * @param[in] decap
3447  *   Pointer to the decap action.
3448  * @param[in] encap
3449  *   Pointer to the encap action.
3450  * @param[in] attr
3451  *   Pointer to flow attributes
3452  * @param[in/out] action_flags
3453  *   Holds the actions detected until now.
3454  * @param[out] actions_n
3455  *   pointer to the number of actions counter.
3456  * @param[in] action
3457  *   Pointer to the action structure.
3458  * @param[in] item_flags
3459  *   Holds the items detected.
3460  * @param[out] error
3461  *   Pointer to error structure.
3462  *
3463  * @return
3464  *   0 on success, a negative errno value otherwise and rte_errno is set.
3465  */
3466 static int
3467 flow_dv_validate_action_raw_encap_decap
3468         (struct rte_eth_dev *dev,
3469          const struct rte_flow_action_raw_decap *decap,
3470          const struct rte_flow_action_raw_encap *encap,
3471          const struct rte_flow_attr *attr, uint64_t *action_flags,
3472          int *actions_n, const struct rte_flow_action *action,
3473          uint64_t item_flags, struct rte_flow_error *error)
3474 {
3475         const struct mlx5_priv *priv = dev->data->dev_private;
3476         int ret;
3477
3478         if (encap && (!encap->size || !encap->data))
3479                 return rte_flow_error_set(error, EINVAL,
3480                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3481                                           "raw encap data cannot be empty");
3482         if (decap && encap) {
3483                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
3484                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3485                         /* L3 encap. */
3486                         decap = NULL;
3487                 else if (encap->size <=
3488                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3489                            decap->size >
3490                            MLX5_ENCAPSULATION_DECISION_SIZE)
3491                         /* L3 decap. */
3492                         encap = NULL;
3493                 else if (encap->size >
3494                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3495                            decap->size >
3496                            MLX5_ENCAPSULATION_DECISION_SIZE)
3497                         /* 2 L2 actions: encap and decap. */
3498                         ;
3499                 else
3500                         return rte_flow_error_set(error,
3501                                 ENOTSUP,
3502                                 RTE_FLOW_ERROR_TYPE_ACTION,
3503                                 NULL, "unsupported too small "
3504                                 "raw decap and too small raw "
3505                                 "encap combination");
3506         }
3507         if (decap) {
3508                 ret = flow_dv_validate_action_decap(dev, *action_flags, action,
3509                                                     item_flags, attr, error);
3510                 if (ret < 0)
3511                         return ret;
3512                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
3513                 ++(*actions_n);
3514         }
3515         if (encap) {
3516                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
3517                         return rte_flow_error_set(error, ENOTSUP,
3518                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3519                                                   NULL,
3520                                                   "small raw encap size");
3521                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
3522                         return rte_flow_error_set(error, EINVAL,
3523                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3524                                                   NULL,
3525                                                   "more than one encap action");
3526                 if (!attr->transfer && priv->representor)
3527                         return rte_flow_error_set
3528                                         (error, ENOTSUP,
3529                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3530                                          "encap action for VF representor "
3531                                          "not supported on NIC table");
3532                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
3533                 ++(*actions_n);
3534         }
3535         return 0;
3536 }
3537
3538 /*
3539  * Validate the ASO CT action.
3540  *
3541  * @param[in] dev
3542  *   Pointer to the rte_eth_dev structure.
3543  * @param[in] action_flags
3544  *   Holds the actions detected until now.
3545  * @param[in] item_flags
3546  *   The items found in this flow rule.
3547  * @param[in] attr
3548  *   Pointer to flow attributes.
3549  * @param[out] error
3550  *   Pointer to error structure.
3551  *
3552  * @return
3553  *   0 on success, a negative errno value otherwise and rte_errno is set.
3554  */
3555 static int
3556 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
3557                                uint64_t action_flags,
3558                                uint64_t item_flags,
3559                                const struct rte_flow_attr *attr,
3560                                struct rte_flow_error *error)
3561 {
3562         RTE_SET_USED(dev);
3563
3564         if (attr->group == 0 && !attr->transfer)
3565                 return rte_flow_error_set(error, ENOTSUP,
3566                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3567                                           NULL,
3568                                           "Only support non-root table");
3569         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
3570                 return rte_flow_error_set(error, ENOTSUP,
3571                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3572                                           "CT cannot follow a fate action");
3573         if ((action_flags & MLX5_FLOW_ACTION_METER) ||
3574             (action_flags & MLX5_FLOW_ACTION_AGE))
3575                 return rte_flow_error_set(error, EINVAL,
3576                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3577                                           "Only one ASO action is supported");
3578         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3579                 return rte_flow_error_set(error, EINVAL,
3580                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3581                                           "Encap cannot exist before CT");
3582         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3583                 return rte_flow_error_set(error, EINVAL,
3584                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3585                                           "Not a outer TCP packet");
3586         return 0;
3587 }
3588
3589 /**
3590  * Match encap_decap resource.
3591  *
3592  * @param list
3593  *   Pointer to the hash list.
3594  * @param entry
3595  *   Pointer to exist resource entry object.
3596  * @param key
3597  *   Key of the new entry.
3598  * @param ctx_cb
3599  *   Pointer to new encap_decap resource.
3600  *
3601  * @return
3602  *   0 on matching, none-zero otherwise.
3603  */
3604 int
3605 flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
3606                              struct mlx5_hlist_entry *entry,
3607                              uint64_t key __rte_unused, void *cb_ctx)
3608 {
3609         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3610         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3611         struct mlx5_flow_dv_encap_decap_resource *resource;
3612
3613         resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
3614                                 entry);
3615         if (resource->reformat_type == ctx_resource->reformat_type &&
3616             resource->ft_type == ctx_resource->ft_type &&
3617             resource->flags == ctx_resource->flags &&
3618             resource->size == ctx_resource->size &&
3619             !memcmp((const void *)resource->buf,
3620                     (const void *)ctx_resource->buf,
3621                     resource->size))
3622                 return 0;
3623         return -1;
3624 }
3625
3626 /**
3627  * Allocate encap_decap resource.
3628  *
3629  * @param list
3630  *   Pointer to the hash list.
3631  * @param entry
3632  *   Pointer to exist resource entry object.
3633  * @param ctx_cb
3634  *   Pointer to new encap_decap resource.
3635  *
3636  * @return
3637  *   0 on matching, none-zero otherwise.
3638  */
3639 struct mlx5_hlist_entry *
3640 flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
3641                               uint64_t key __rte_unused,
3642                               void *cb_ctx)
3643 {
3644         struct mlx5_dev_ctx_shared *sh = list->ctx;
3645         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3646         struct mlx5dv_dr_domain *domain;
3647         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3648         struct mlx5_flow_dv_encap_decap_resource *resource;
3649         uint32_t idx;
3650         int ret;
3651
3652         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3653                 domain = sh->fdb_domain;
3654         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3655                 domain = sh->rx_domain;
3656         else
3657                 domain = sh->tx_domain;
3658         /* Register new encap/decap resource. */
3659         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
3660         if (!resource) {
3661                 rte_flow_error_set(ctx->error, ENOMEM,
3662                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3663                                    "cannot allocate resource memory");
3664                 return NULL;
3665         }
3666         *resource = *ctx_resource;
3667         resource->idx = idx;
3668         ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain,
3669                                                               resource,
3670                                                              &resource->action);
3671         if (ret) {
3672                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
3673                 rte_flow_error_set(ctx->error, ENOMEM,
3674                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3675                                    NULL, "cannot create action");
3676                 return NULL;
3677         }
3678
3679         return &resource->entry;
3680 }
3681
3682 /**
3683  * Find existing encap/decap resource or create and register a new one.
3684  *
3685  * @param[in, out] dev
3686  *   Pointer to rte_eth_dev structure.
3687  * @param[in, out] resource
3688  *   Pointer to encap/decap resource.
3689  * @parm[in, out] dev_flow
3690  *   Pointer to the dev_flow.
3691  * @param[out] error
3692  *   pointer to error structure.
3693  *
3694  * @return
3695  *   0 on success otherwise -errno and errno is set.
3696  */
3697 static int
3698 flow_dv_encap_decap_resource_register
3699                         (struct rte_eth_dev *dev,
3700                          struct mlx5_flow_dv_encap_decap_resource *resource,
3701                          struct mlx5_flow *dev_flow,
3702                          struct rte_flow_error *error)
3703 {
3704         struct mlx5_priv *priv = dev->data->dev_private;
3705         struct mlx5_dev_ctx_shared *sh = priv->sh;
3706         struct mlx5_hlist_entry *entry;
3707         union {
3708                 struct {
3709                         uint32_t ft_type:8;
3710                         uint32_t refmt_type:8;
3711                         /*
3712                          * Header reformat actions can be shared between
3713                          * non-root tables. One bit to indicate non-root
3714                          * table or not.
3715                          */
3716                         uint32_t is_root:1;
3717                         uint32_t reserve:15;
3718                 };
3719                 uint32_t v32;
3720         } encap_decap_key = {
3721                 {
3722                         .ft_type = resource->ft_type,
3723                         .refmt_type = resource->reformat_type,
3724                         .is_root = !!dev_flow->dv.group,
3725                         .reserve = 0,
3726                 }
3727         };
3728         struct mlx5_flow_cb_ctx ctx = {
3729                 .error = error,
3730                 .data = resource,
3731         };
3732         uint64_t key64;
3733
3734         resource->flags = dev_flow->dv.group ? 0 : 1;
3735         key64 =  __rte_raw_cksum(&encap_decap_key.v32,
3736                                  sizeof(encap_decap_key.v32), 0);
3737         if (resource->reformat_type !=
3738             MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
3739             resource->size)
3740                 key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
3741         entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
3742         if (!entry)
3743                 return -rte_errno;
3744         resource = container_of(entry, typeof(*resource), entry);
3745         dev_flow->dv.encap_decap = resource;
3746         dev_flow->handle->dvh.rix_encap_decap = resource->idx;
3747         return 0;
3748 }
3749
3750 /**
3751  * Find existing table jump resource or create and register a new one.
3752  *
3753  * @param[in, out] dev
3754  *   Pointer to rte_eth_dev structure.
3755  * @param[in, out] tbl
3756  *   Pointer to flow table resource.
3757  * @parm[in, out] dev_flow
3758  *   Pointer to the dev_flow.
3759  * @param[out] error
3760  *   pointer to error structure.
3761  *
3762  * @return
3763  *   0 on success otherwise -errno and errno is set.
3764  */
3765 static int
3766 flow_dv_jump_tbl_resource_register
3767                         (struct rte_eth_dev *dev __rte_unused,
3768                          struct mlx5_flow_tbl_resource *tbl,
3769                          struct mlx5_flow *dev_flow,
3770                          struct rte_flow_error *error __rte_unused)
3771 {
3772         struct mlx5_flow_tbl_data_entry *tbl_data =
3773                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
3774
3775         MLX5_ASSERT(tbl);
3776         MLX5_ASSERT(tbl_data->jump.action);
3777         dev_flow->handle->rix_jump = tbl_data->idx;
3778         dev_flow->dv.jump = &tbl_data->jump;
3779         return 0;
3780 }
3781
3782 int
3783 flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
3784                          struct mlx5_list_entry *entry, void *cb_ctx)
3785 {
3786         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3787         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3788         struct mlx5_flow_dv_port_id_action_resource *res =
3789                                        container_of(entry, typeof(*res), entry);
3790
3791         return ref->port_id != res->port_id;
3792 }
3793
3794 struct mlx5_list_entry *
3795 flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
3796 {
3797         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3798         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3799         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3800         struct mlx5_flow_dv_port_id_action_resource *resource;
3801         uint32_t idx;
3802         int ret;
3803
3804         /* Register new port id action resource. */
3805         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3806         if (!resource) {
3807                 rte_flow_error_set(ctx->error, ENOMEM,
3808                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3809                                    "cannot allocate port_id action memory");
3810                 return NULL;
3811         }
3812         *resource = *ref;
3813         ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
3814                                                         ref->port_id,
3815                                                         &resource->action);
3816         if (ret) {
3817                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
3818                 rte_flow_error_set(ctx->error, ENOMEM,
3819                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3820                                    "cannot create action");
3821                 return NULL;
3822         }
3823         resource->idx = idx;
3824         return &resource->entry;
3825 }
3826
3827 struct mlx5_list_entry *
3828 flow_dv_port_id_clone_cb(void *tool_ctx,
3829                          struct mlx5_list_entry *entry __rte_unused,
3830                          void *cb_ctx)
3831 {
3832         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3833         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3834         struct mlx5_flow_dv_port_id_action_resource *resource;
3835         uint32_t idx;
3836
3837         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3838         if (!resource) {
3839                 rte_flow_error_set(ctx->error, ENOMEM,
3840                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3841                                    "cannot allocate port_id action memory");
3842                 return NULL;
3843         }
3844         memcpy(resource, entry, sizeof(*resource));
3845         resource->idx = idx;
3846         return &resource->entry;
3847 }
3848
3849 void
3850 flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3851 {
3852         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3853         struct mlx5_flow_dv_port_id_action_resource *resource =
3854                                   container_of(entry, typeof(*resource), entry);
3855
3856         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
3857 }
3858
3859 /**
3860  * Find existing table port ID resource or create and register a new one.
3861  *
3862  * @param[in, out] dev
3863  *   Pointer to rte_eth_dev structure.
3864  * @param[in, out] ref
3865  *   Pointer to port ID action resource reference.
3866  * @parm[in, out] dev_flow
3867  *   Pointer to the dev_flow.
3868  * @param[out] error
3869  *   pointer to error structure.
3870  *
3871  * @return
3872  *   0 on success otherwise -errno and errno is set.
3873  */
3874 static int
3875 flow_dv_port_id_action_resource_register
3876                         (struct rte_eth_dev *dev,
3877                          struct mlx5_flow_dv_port_id_action_resource *ref,
3878                          struct mlx5_flow *dev_flow,
3879                          struct rte_flow_error *error)
3880 {
3881         struct mlx5_priv *priv = dev->data->dev_private;
3882         struct mlx5_list_entry *entry;
3883         struct mlx5_flow_dv_port_id_action_resource *resource;
3884         struct mlx5_flow_cb_ctx ctx = {
3885                 .error = error,
3886                 .data = ref,
3887         };
3888
3889         entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
3890         if (!entry)
3891                 return -rte_errno;
3892         resource = container_of(entry, typeof(*resource), entry);
3893         dev_flow->dv.port_id_action = resource;
3894         dev_flow->handle->rix_port_id_action = resource->idx;
3895         return 0;
3896 }
3897
3898 int
3899 flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
3900                            struct mlx5_list_entry *entry, void *cb_ctx)
3901 {
3902         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3903         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3904         struct mlx5_flow_dv_push_vlan_action_resource *res =
3905                                        container_of(entry, typeof(*res), entry);
3906
3907         return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
3908 }
3909
3910 struct mlx5_list_entry *
3911 flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
3912 {
3913         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3914         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3915         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3916         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3917         struct mlx5dv_dr_domain *domain;
3918         uint32_t idx;
3919         int ret;
3920
3921         /* Register new port id action resource. */
3922         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3923         if (!resource) {
3924                 rte_flow_error_set(ctx->error, ENOMEM,
3925                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3926                                    "cannot allocate push_vlan action memory");
3927                 return NULL;
3928         }
3929         *resource = *ref;
3930         if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3931                 domain = sh->fdb_domain;
3932         else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3933                 domain = sh->rx_domain;
3934         else
3935                 domain = sh->tx_domain;
3936         ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
3937                                                         &resource->action);
3938         if (ret) {
3939                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
3940                 rte_flow_error_set(ctx->error, ENOMEM,
3941                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3942                                    "cannot create push vlan action");
3943                 return NULL;
3944         }
3945         resource->idx = idx;
3946         return &resource->entry;
3947 }
3948
3949 struct mlx5_list_entry *
3950 flow_dv_push_vlan_clone_cb(void *tool_ctx,
3951                            struct mlx5_list_entry *entry __rte_unused,
3952                            void *cb_ctx)
3953 {
3954         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3955         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3956         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3957         uint32_t idx;
3958
3959         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3960         if (!resource) {
3961                 rte_flow_error_set(ctx->error, ENOMEM,
3962                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3963                                    "cannot allocate push_vlan action memory");
3964                 return NULL;
3965         }
3966         memcpy(resource, entry, sizeof(*resource));
3967         resource->idx = idx;
3968         return &resource->entry;
3969 }
3970
3971 void
3972 flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3973 {
3974         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3975         struct mlx5_flow_dv_push_vlan_action_resource *resource =
3976                                   container_of(entry, typeof(*resource), entry);
3977
3978         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
3979 }
3980
3981 /**
3982  * Find existing push vlan resource or create and register a new one.
3983  *
3984  * @param [in, out] dev
3985  *   Pointer to rte_eth_dev structure.
3986  * @param[in, out] ref
3987  *   Pointer to port ID action resource reference.
3988  * @parm[in, out] dev_flow
3989  *   Pointer to the dev_flow.
3990  * @param[out] error
3991  *   pointer to error structure.
3992  *
3993  * @return
3994  *   0 on success otherwise -errno and errno is set.
3995  */
3996 static int
3997 flow_dv_push_vlan_action_resource_register
3998                        (struct rte_eth_dev *dev,
3999                         struct mlx5_flow_dv_push_vlan_action_resource *ref,
4000                         struct mlx5_flow *dev_flow,
4001                         struct rte_flow_error *error)
4002 {
4003         struct mlx5_priv *priv = dev->data->dev_private;
4004         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4005         struct mlx5_list_entry *entry;
4006         struct mlx5_flow_cb_ctx ctx = {
4007                 .error = error,
4008                 .data = ref,
4009         };
4010
4011         entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
4012         if (!entry)
4013                 return -rte_errno;
4014         resource = container_of(entry, typeof(*resource), entry);
4015
4016         dev_flow->handle->dvh.rix_push_vlan = resource->idx;
4017         dev_flow->dv.push_vlan_res = resource;
4018         return 0;
4019 }
4020
4021 /**
4022  * Get the size of specific rte_flow_item_type hdr size
4023  *
4024  * @param[in] item_type
4025  *   Tested rte_flow_item_type.
4026  *
4027  * @return
4028  *   sizeof struct item_type, 0 if void or irrelevant.
4029  */
4030 static size_t
4031 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
4032 {
4033         size_t retval;
4034
4035         switch (item_type) {
4036         case RTE_FLOW_ITEM_TYPE_ETH:
4037                 retval = sizeof(struct rte_ether_hdr);
4038                 break;
4039         case RTE_FLOW_ITEM_TYPE_VLAN:
4040                 retval = sizeof(struct rte_vlan_hdr);
4041                 break;
4042         case RTE_FLOW_ITEM_TYPE_IPV4:
4043                 retval = sizeof(struct rte_ipv4_hdr);
4044                 break;
4045         case RTE_FLOW_ITEM_TYPE_IPV6:
4046                 retval = sizeof(struct rte_ipv6_hdr);
4047                 break;
4048         case RTE_FLOW_ITEM_TYPE_UDP:
4049                 retval = sizeof(struct rte_udp_hdr);
4050                 break;
4051         case RTE_FLOW_ITEM_TYPE_TCP:
4052                 retval = sizeof(struct rte_tcp_hdr);
4053                 break;
4054         case RTE_FLOW_ITEM_TYPE_VXLAN:
4055         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4056                 retval = sizeof(struct rte_vxlan_hdr);
4057                 break;
4058         case RTE_FLOW_ITEM_TYPE_GRE:
4059         case RTE_FLOW_ITEM_TYPE_NVGRE:
4060                 retval = sizeof(struct rte_gre_hdr);
4061                 break;
4062         case RTE_FLOW_ITEM_TYPE_MPLS:
4063                 retval = sizeof(struct rte_mpls_hdr);
4064                 break;
4065         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
4066         default:
4067                 retval = 0;
4068                 break;
4069         }
4070         return retval;
4071 }
4072
4073 #define MLX5_ENCAP_IPV4_VERSION         0x40
4074 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
4075 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
4076 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
4077 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
4078 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
4079 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
4080
4081 /**
4082  * Convert the encap action data from list of rte_flow_item to raw buffer
4083  *
4084  * @param[in] items
4085  *   Pointer to rte_flow_item objects list.
4086  * @param[out] buf
4087  *   Pointer to the output buffer.
4088  * @param[out] size
4089  *   Pointer to the output buffer size.
4090  * @param[out] error
4091  *   Pointer to the error structure.
4092  *
4093  * @return
4094  *   0 on success, a negative errno value otherwise and rte_errno is set.
4095  */
4096 static int
4097 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
4098                            size_t *size, struct rte_flow_error *error)
4099 {
4100         struct rte_ether_hdr *eth = NULL;
4101         struct rte_vlan_hdr *vlan = NULL;
4102         struct rte_ipv4_hdr *ipv4 = NULL;
4103         struct rte_ipv6_hdr *ipv6 = NULL;
4104         struct rte_udp_hdr *udp = NULL;
4105         struct rte_vxlan_hdr *vxlan = NULL;
4106         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
4107         struct rte_gre_hdr *gre = NULL;
4108         size_t len;
4109         size_t temp_size = 0;
4110
4111         if (!items)
4112                 return rte_flow_error_set(error, EINVAL,
4113                                           RTE_FLOW_ERROR_TYPE_ACTION,
4114                                           NULL, "invalid empty data");
4115         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4116                 len = flow_dv_get_item_hdr_len(items->type);
4117                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
4118                         return rte_flow_error_set(error, EINVAL,
4119                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4120                                                   (void *)items->type,
4121                                                   "items total size is too big"
4122                                                   " for encap action");
4123                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
4124                 switch (items->type) {
4125                 case RTE_FLOW_ITEM_TYPE_ETH:
4126                         eth = (struct rte_ether_hdr *)&buf[temp_size];
4127                         break;
4128                 case RTE_FLOW_ITEM_TYPE_VLAN:
4129                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
4130                         if (!eth)
4131                                 return rte_flow_error_set(error, EINVAL,
4132                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4133                                                 (void *)items->type,
4134                                                 "eth header not found");
4135                         if (!eth->ether_type)
4136                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
4137                         break;
4138                 case RTE_FLOW_ITEM_TYPE_IPV4:
4139                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
4140                         if (!vlan && !eth)
4141                                 return rte_flow_error_set(error, EINVAL,
4142                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4143                                                 (void *)items->type,
4144                                                 "neither eth nor vlan"
4145                                                 " header found");
4146                         if (vlan && !vlan->eth_proto)
4147                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4148                         else if (eth && !eth->ether_type)
4149                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4150                         if (!ipv4->version_ihl)
4151                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
4152                                                     MLX5_ENCAP_IPV4_IHL_MIN;
4153                         if (!ipv4->time_to_live)
4154                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
4155                         break;
4156                 case RTE_FLOW_ITEM_TYPE_IPV6:
4157                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
4158                         if (!vlan && !eth)
4159                                 return rte_flow_error_set(error, EINVAL,
4160                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4161                                                 (void *)items->type,
4162                                                 "neither eth nor vlan"
4163                                                 " header found");
4164                         if (vlan && !vlan->eth_proto)
4165                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4166                         else if (eth && !eth->ether_type)
4167                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4168                         if (!ipv6->vtc_flow)
4169                                 ipv6->vtc_flow =
4170                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
4171                         if (!ipv6->hop_limits)
4172                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
4173                         break;
4174                 case RTE_FLOW_ITEM_TYPE_UDP:
4175                         udp = (struct rte_udp_hdr *)&buf[temp_size];
4176                         if (!ipv4 && !ipv6)
4177                                 return rte_flow_error_set(error, EINVAL,
4178                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4179                                                 (void *)items->type,
4180                                                 "ip header not found");
4181                         if (ipv4 && !ipv4->next_proto_id)
4182                                 ipv4->next_proto_id = IPPROTO_UDP;
4183                         else if (ipv6 && !ipv6->proto)
4184                                 ipv6->proto = IPPROTO_UDP;
4185                         break;
4186                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4187                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
4188                         if (!udp)
4189                                 return rte_flow_error_set(error, EINVAL,
4190                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4191                                                 (void *)items->type,
4192                                                 "udp header not found");
4193                         if (!udp->dst_port)
4194                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
4195                         if (!vxlan->vx_flags)
4196                                 vxlan->vx_flags =
4197                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
4198                         break;
4199                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4200                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
4201                         if (!udp)
4202                                 return rte_flow_error_set(error, EINVAL,
4203                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4204                                                 (void *)items->type,
4205                                                 "udp header not found");
4206                         if (!vxlan_gpe->proto)
4207                                 return rte_flow_error_set(error, EINVAL,
4208                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4209                                                 (void *)items->type,
4210                                                 "next protocol not found");
4211                         if (!udp->dst_port)
4212                                 udp->dst_port =
4213                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
4214                         if (!vxlan_gpe->vx_flags)
4215                                 vxlan_gpe->vx_flags =
4216                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
4217                         break;
4218                 case RTE_FLOW_ITEM_TYPE_GRE:
4219                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4220                         gre = (struct rte_gre_hdr *)&buf[temp_size];
4221                         if (!gre->proto)
4222                                 return rte_flow_error_set(error, EINVAL,
4223                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4224                                                 (void *)items->type,
4225                                                 "next protocol not found");
4226                         if (!ipv4 && !ipv6)
4227                                 return rte_flow_error_set(error, EINVAL,
4228                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4229                                                 (void *)items->type,
4230                                                 "ip header not found");
4231                         if (ipv4 && !ipv4->next_proto_id)
4232                                 ipv4->next_proto_id = IPPROTO_GRE;
4233                         else if (ipv6 && !ipv6->proto)
4234                                 ipv6->proto = IPPROTO_GRE;
4235                         break;
4236                 case RTE_FLOW_ITEM_TYPE_VOID:
4237                         break;
4238                 default:
4239                         return rte_flow_error_set(error, EINVAL,
4240                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4241                                                   (void *)items->type,
4242                                                   "unsupported item type");
4243                         break;
4244                 }
4245                 temp_size += len;
4246         }
4247         *size = temp_size;
4248         return 0;
4249 }
4250
4251 static int
4252 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
4253 {
4254         struct rte_ether_hdr *eth = NULL;
4255         struct rte_vlan_hdr *vlan = NULL;
4256         struct rte_ipv6_hdr *ipv6 = NULL;
4257         struct rte_udp_hdr *udp = NULL;
4258         char *next_hdr;
4259         uint16_t proto;
4260
4261         eth = (struct rte_ether_hdr *)data;
4262         next_hdr = (char *)(eth + 1);
4263         proto = RTE_BE16(eth->ether_type);
4264
4265         /* VLAN skipping */
4266         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
4267                 vlan = (struct rte_vlan_hdr *)next_hdr;
4268                 proto = RTE_BE16(vlan->eth_proto);
4269                 next_hdr += sizeof(struct rte_vlan_hdr);
4270         }
4271
4272         /* HW calculates IPv4 csum. no need to proceed */
4273         if (proto == RTE_ETHER_TYPE_IPV4)
4274                 return 0;
4275
4276         /* non IPv4/IPv6 header. not supported */
4277         if (proto != RTE_ETHER_TYPE_IPV6) {
4278                 return rte_flow_error_set(error, ENOTSUP,
4279                                           RTE_FLOW_ERROR_TYPE_ACTION,
4280                                           NULL, "Cannot offload non IPv4/IPv6");
4281         }
4282
4283         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
4284
4285         /* ignore non UDP */
4286         if (ipv6->proto != IPPROTO_UDP)
4287                 return 0;
4288
4289         udp = (struct rte_udp_hdr *)(ipv6 + 1);
4290         udp->dgram_cksum = 0;
4291
4292         return 0;
4293 }
4294
4295 /**
4296  * Convert L2 encap action to DV specification.
4297  *
4298  * @param[in] dev
4299  *   Pointer to rte_eth_dev structure.
4300  * @param[in] action
4301  *   Pointer to action structure.
4302  * @param[in, out] dev_flow
4303  *   Pointer to the mlx5_flow.
4304  * @param[in] transfer
4305  *   Mark if the flow is E-Switch flow.
4306  * @param[out] error
4307  *   Pointer to the error structure.
4308  *
4309  * @return
4310  *   0 on success, a negative errno value otherwise and rte_errno is set.
4311  */
4312 static int
4313 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
4314                                const struct rte_flow_action *action,
4315                                struct mlx5_flow *dev_flow,
4316                                uint8_t transfer,
4317                                struct rte_flow_error *error)
4318 {
4319         const struct rte_flow_item *encap_data;
4320         const struct rte_flow_action_raw_encap *raw_encap_data;
4321         struct mlx5_flow_dv_encap_decap_resource res = {
4322                 .reformat_type =
4323                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
4324                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4325                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
4326         };
4327
4328         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4329                 raw_encap_data =
4330                         (const struct rte_flow_action_raw_encap *)action->conf;
4331                 res.size = raw_encap_data->size;
4332                 memcpy(res.buf, raw_encap_data->data, res.size);
4333         } else {
4334                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
4335                         encap_data =
4336                                 ((const struct rte_flow_action_vxlan_encap *)
4337                                                 action->conf)->definition;
4338                 else
4339                         encap_data =
4340                                 ((const struct rte_flow_action_nvgre_encap *)
4341                                                 action->conf)->definition;
4342                 if (flow_dv_convert_encap_data(encap_data, res.buf,
4343                                                &res.size, error))
4344                         return -rte_errno;
4345         }
4346         if (flow_dv_zero_encap_udp_csum(res.buf, error))
4347                 return -rte_errno;
4348         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4349                 return rte_flow_error_set(error, EINVAL,
4350                                           RTE_FLOW_ERROR_TYPE_ACTION,
4351                                           NULL, "can't create L2 encap action");
4352         return 0;
4353 }
4354
4355 /**
4356  * Convert L2 decap action to DV specification.
4357  *
4358  * @param[in] dev
4359  *   Pointer to rte_eth_dev structure.
4360  * @param[in, out] dev_flow
4361  *   Pointer to the mlx5_flow.
4362  * @param[in] transfer
4363  *   Mark if the flow is E-Switch flow.
4364  * @param[out] error
4365  *   Pointer to the error structure.
4366  *
4367  * @return
4368  *   0 on success, a negative errno value otherwise and rte_errno is set.
4369  */
4370 static int
4371 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
4372                                struct mlx5_flow *dev_flow,
4373                                uint8_t transfer,
4374                                struct rte_flow_error *error)
4375 {
4376         struct mlx5_flow_dv_encap_decap_resource res = {
4377                 .size = 0,
4378                 .reformat_type =
4379                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
4380                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4381                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
4382         };
4383
4384         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4385                 return rte_flow_error_set(error, EINVAL,
4386                                           RTE_FLOW_ERROR_TYPE_ACTION,
4387                                           NULL, "can't create L2 decap action");
4388         return 0;
4389 }
4390
4391 /**
4392  * Convert raw decap/encap (L3 tunnel) action to DV specification.
4393  *
4394  * @param[in] dev
4395  *   Pointer to rte_eth_dev structure.
4396  * @param[in] action
4397  *   Pointer to action structure.
4398  * @param[in, out] dev_flow
4399  *   Pointer to the mlx5_flow.
4400  * @param[in] attr
4401  *   Pointer to the flow attributes.
4402  * @param[out] error
4403  *   Pointer to the error structure.
4404  *
4405  * @return
4406  *   0 on success, a negative errno value otherwise and rte_errno is set.
4407  */
4408 static int
4409 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
4410                                 const struct rte_flow_action *action,
4411                                 struct mlx5_flow *dev_flow,
4412                                 const struct rte_flow_attr *attr,
4413                                 struct rte_flow_error *error)
4414 {
4415         const struct rte_flow_action_raw_encap *encap_data;
4416         struct mlx5_flow_dv_encap_decap_resource res;
4417
4418         memset(&res, 0, sizeof(res));
4419         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
4420         res.size = encap_data->size;
4421         memcpy(res.buf, encap_data->data, res.size);
4422         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
4423                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
4424                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
4425         if (attr->transfer)
4426                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4427         else
4428                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4429                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4430         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4431                 return rte_flow_error_set(error, EINVAL,
4432                                           RTE_FLOW_ERROR_TYPE_ACTION,
4433                                           NULL, "can't create encap action");
4434         return 0;
4435 }
4436
4437 /**
4438  * Create action push VLAN.
4439  *
4440  * @param[in] dev
4441  *   Pointer to rte_eth_dev structure.
4442  * @param[in] attr
4443  *   Pointer to the flow attributes.
4444  * @param[in] vlan
4445  *   Pointer to the vlan to push to the Ethernet header.
4446  * @param[in, out] dev_flow
4447  *   Pointer to the mlx5_flow.
4448  * @param[out] error
4449  *   Pointer to the error structure.
4450  *
4451  * @return
4452  *   0 on success, a negative errno value otherwise and rte_errno is set.
4453  */
4454 static int
4455 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
4456                                 const struct rte_flow_attr *attr,
4457                                 const struct rte_vlan_hdr *vlan,
4458                                 struct mlx5_flow *dev_flow,
4459                                 struct rte_flow_error *error)
4460 {
4461         struct mlx5_flow_dv_push_vlan_action_resource res;
4462
4463         memset(&res, 0, sizeof(res));
4464         res.vlan_tag =
4465                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
4466                                  vlan->vlan_tci);
4467         if (attr->transfer)
4468                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4469         else
4470                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4471                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4472         return flow_dv_push_vlan_action_resource_register
4473                                             (dev, &res, dev_flow, error);
4474 }
4475
4476 /**
4477  * Validate the modify-header actions.
4478  *
4479  * @param[in] action_flags
4480  *   Holds the actions detected until now.
4481  * @param[in] action
4482  *   Pointer to the modify action.
4483  * @param[out] error
4484  *   Pointer to error structure.
4485  *
4486  * @return
4487  *   0 on success, a negative errno value otherwise and rte_errno is set.
4488  */
4489 static int
4490 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
4491                                    const struct rte_flow_action *action,
4492                                    struct rte_flow_error *error)
4493 {
4494         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
4495                 return rte_flow_error_set(error, EINVAL,
4496                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4497                                           NULL, "action configuration not set");
4498         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
4499                 return rte_flow_error_set(error, EINVAL,
4500                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4501                                           "can't have encap action before"
4502                                           " modify action");
4503         return 0;
4504 }
4505
4506 /**
4507  * Validate the modify-header MAC address actions.
4508  *
4509  * @param[in] action_flags
4510  *   Holds the actions detected until now.
4511  * @param[in] action
4512  *   Pointer to the modify action.
4513  * @param[in] item_flags
4514  *   Holds the items detected.
4515  * @param[out] error
4516  *   Pointer to error structure.
4517  *
4518  * @return
4519  *   0 on success, a negative errno value otherwise and rte_errno is set.
4520  */
4521 static int
4522 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
4523                                    const struct rte_flow_action *action,
4524                                    const uint64_t item_flags,
4525                                    struct rte_flow_error *error)
4526 {
4527         int ret = 0;
4528
4529         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4530         if (!ret) {
4531                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
4532                         return rte_flow_error_set(error, EINVAL,
4533                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4534                                                   NULL,
4535                                                   "no L2 item in pattern");
4536         }
4537         return ret;
4538 }
4539
4540 /**
4541  * Validate the modify-header IPv4 address actions.
4542  *
4543  * @param[in] action_flags
4544  *   Holds the actions detected until now.
4545  * @param[in] action
4546  *   Pointer to the modify action.
4547  * @param[in] item_flags
4548  *   Holds the items detected.
4549  * @param[out] error
4550  *   Pointer to error structure.
4551  *
4552  * @return
4553  *   0 on success, a negative errno value otherwise and rte_errno is set.
4554  */
4555 static int
4556 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
4557                                     const struct rte_flow_action *action,
4558                                     const uint64_t item_flags,
4559                                     struct rte_flow_error *error)
4560 {
4561         int ret = 0;
4562         uint64_t layer;
4563
4564         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4565         if (!ret) {
4566                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4567                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4568                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4569                 if (!(item_flags & layer))
4570                         return rte_flow_error_set(error, EINVAL,
4571                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4572                                                   NULL,
4573                                                   "no ipv4 item in pattern");
4574         }
4575         return ret;
4576 }
4577
4578 /**
4579  * Validate the modify-header IPv6 address actions.
4580  *
4581  * @param[in] action_flags
4582  *   Holds the actions detected until now.
4583  * @param[in] action
4584  *   Pointer to the modify action.
4585  * @param[in] item_flags
4586  *   Holds the items detected.
4587  * @param[out] error
4588  *   Pointer to error structure.
4589  *
4590  * @return
4591  *   0 on success, a negative errno value otherwise and rte_errno is set.
4592  */
4593 static int
4594 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
4595                                     const struct rte_flow_action *action,
4596                                     const uint64_t item_flags,
4597                                     struct rte_flow_error *error)
4598 {
4599         int ret = 0;
4600         uint64_t layer;
4601
4602         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4603         if (!ret) {
4604                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4605                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4606                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4607                 if (!(item_flags & layer))
4608                         return rte_flow_error_set(error, EINVAL,
4609                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4610                                                   NULL,
4611                                                   "no ipv6 item in pattern");
4612         }
4613         return ret;
4614 }
4615
4616 /**
4617  * Validate the modify-header TP actions.
4618  *
4619  * @param[in] action_flags
4620  *   Holds the actions detected until now.
4621  * @param[in] action
4622  *   Pointer to the modify action.
4623  * @param[in] item_flags
4624  *   Holds the items detected.
4625  * @param[out] error
4626  *   Pointer to error structure.
4627  *
4628  * @return
4629  *   0 on success, a negative errno value otherwise and rte_errno is set.
4630  */
4631 static int
4632 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
4633                                   const struct rte_flow_action *action,
4634                                   const uint64_t item_flags,
4635                                   struct rte_flow_error *error)
4636 {
4637         int ret = 0;
4638         uint64_t layer;
4639
4640         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4641         if (!ret) {
4642                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4643                                  MLX5_FLOW_LAYER_INNER_L4 :
4644                                  MLX5_FLOW_LAYER_OUTER_L4;
4645                 if (!(item_flags & layer))
4646                         return rte_flow_error_set(error, EINVAL,
4647                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4648                                                   NULL, "no transport layer "
4649                                                   "in pattern");
4650         }
4651         return ret;
4652 }
4653
4654 /**
4655  * Validate the modify-header actions of increment/decrement
4656  * TCP Sequence-number.
4657  *
4658  * @param[in] action_flags
4659  *   Holds the actions detected until now.
4660  * @param[in] action
4661  *   Pointer to the modify action.
4662  * @param[in] item_flags
4663  *   Holds the items detected.
4664  * @param[out] error
4665  *   Pointer to error structure.
4666  *
4667  * @return
4668  *   0 on success, a negative errno value otherwise and rte_errno is set.
4669  */
4670 static int
4671 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
4672                                        const struct rte_flow_action *action,
4673                                        const uint64_t item_flags,
4674                                        struct rte_flow_error *error)
4675 {
4676         int ret = 0;
4677         uint64_t layer;
4678
4679         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4680         if (!ret) {
4681                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4682                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4683                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4684                 if (!(item_flags & layer))
4685                         return rte_flow_error_set(error, EINVAL,
4686                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4687                                                   NULL, "no TCP item in"
4688                                                   " pattern");
4689                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
4690                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
4691                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
4692                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
4693                         return rte_flow_error_set(error, EINVAL,
4694                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4695                                                   NULL,
4696                                                   "cannot decrease and increase"
4697                                                   " TCP sequence number"
4698                                                   " at the same time");
4699         }
4700         return ret;
4701 }
4702
4703 /**
4704  * Validate the modify-header actions of increment/decrement
4705  * TCP Acknowledgment number.
4706  *
4707  * @param[in] action_flags
4708  *   Holds the actions detected until now.
4709  * @param[in] action
4710  *   Pointer to the modify action.
4711  * @param[in] item_flags
4712  *   Holds the items detected.
4713  * @param[out] error
4714  *   Pointer to error structure.
4715  *
4716  * @return
4717  *   0 on success, a negative errno value otherwise and rte_errno is set.
4718  */
4719 static int
4720 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
4721                                        const struct rte_flow_action *action,
4722                                        const uint64_t item_flags,
4723                                        struct rte_flow_error *error)
4724 {
4725         int ret = 0;
4726         uint64_t layer;
4727
4728         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4729         if (!ret) {
4730                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4731                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4732                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4733                 if (!(item_flags & layer))
4734                         return rte_flow_error_set(error, EINVAL,
4735                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4736                                                   NULL, "no TCP item in"
4737                                                   " pattern");
4738                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
4739                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
4740                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
4741                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
4742                         return rte_flow_error_set(error, EINVAL,
4743                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4744                                                   NULL,
4745                                                   "cannot decrease and increase"
4746                                                   " TCP acknowledgment number"
4747                                                   " at the same time");
4748         }
4749         return ret;
4750 }
4751
4752 /**
4753  * Validate the modify-header TTL actions.
4754  *
4755  * @param[in] action_flags
4756  *   Holds the actions detected until now.
4757  * @param[in] action
4758  *   Pointer to the modify action.
4759  * @param[in] item_flags
4760  *   Holds the items detected.
4761  * @param[out] error
4762  *   Pointer to error structure.
4763  *
4764  * @return
4765  *   0 on success, a negative errno value otherwise and rte_errno is set.
4766  */
4767 static int
4768 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
4769                                    const struct rte_flow_action *action,
4770                                    const uint64_t item_flags,
4771                                    struct rte_flow_error *error)
4772 {
4773         int ret = 0;
4774         uint64_t layer;
4775
4776         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4777         if (!ret) {
4778                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4779                                  MLX5_FLOW_LAYER_INNER_L3 :
4780                                  MLX5_FLOW_LAYER_OUTER_L3;
4781                 if (!(item_flags & layer))
4782                         return rte_flow_error_set(error, EINVAL,
4783                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4784                                                   NULL,
4785                                                   "no IP protocol in pattern");
4786         }
4787         return ret;
4788 }
4789
4790 /**
4791  * Validate the generic modify field actions.
4792  * @param[in] dev
4793  *   Pointer to the rte_eth_dev structure.
4794  * @param[in] action_flags
4795  *   Holds the actions detected until now.
4796  * @param[in] action
4797  *   Pointer to the modify action.
4798  * @param[in] attr
4799  *   Pointer to the flow attributes.
4800  * @param[out] error
4801  *   Pointer to error structure.
4802  *
4803  * @return
4804  *   Number of header fields to modify (0 or more) on success,
4805  *   a negative errno value otherwise and rte_errno is set.
4806  */
4807 static int
4808 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev,
4809                                    const uint64_t action_flags,
4810                                    const struct rte_flow_action *action,
4811                                    const struct rte_flow_attr *attr,
4812                                    struct rte_flow_error *error)
4813 {
4814         int ret = 0;
4815         struct mlx5_priv *priv = dev->data->dev_private;
4816         struct mlx5_dev_config *config = &priv->config;
4817         const struct rte_flow_action_modify_field *action_modify_field =
4818                 action->conf;
4819         uint32_t dst_width = mlx5_flow_item_field_width(config,
4820                                 action_modify_field->dst.field);
4821         uint32_t src_width = mlx5_flow_item_field_width(config,
4822                                 action_modify_field->src.field);
4823
4824         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4825         if (ret)
4826                 return ret;
4827
4828         if (action_modify_field->width == 0)
4829                 return rte_flow_error_set(error, EINVAL,
4830                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4831                                 "no bits are requested to be modified");
4832         else if (action_modify_field->width > dst_width ||
4833                  action_modify_field->width > src_width)
4834                 return rte_flow_error_set(error, EINVAL,
4835                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4836                                 "cannot modify more bits than"
4837                                 " the width of a field");
4838         if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE &&
4839             action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) {
4840                 if ((action_modify_field->dst.offset +
4841                      action_modify_field->width > dst_width) ||
4842                     (action_modify_field->dst.offset % 32))
4843                         return rte_flow_error_set(error, EINVAL,
4844                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4845                                         "destination offset is too big"
4846                                         " or not aligned to 4 bytes");
4847                 if (action_modify_field->dst.level &&
4848                     action_modify_field->dst.field != RTE_FLOW_FIELD_TAG)
4849                         return rte_flow_error_set(error, ENOTSUP,
4850                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4851                                         "inner header fields modification"
4852                                         " is not supported");
4853         }
4854         if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE &&
4855             action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) {
4856                 if (!attr->transfer && !attr->group)
4857                         return rte_flow_error_set(error, ENOTSUP,
4858                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4859                                         "modify field action is not"
4860                                         " supported for group 0");
4861                 if ((action_modify_field->src.offset +
4862                      action_modify_field->width > src_width) ||
4863                     (action_modify_field->src.offset % 32))
4864                         return rte_flow_error_set(error, EINVAL,
4865                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4866                                         "source offset is too big"
4867                                         " or not aligned to 4 bytes");
4868                 if (action_modify_field->src.level &&
4869                     action_modify_field->src.field != RTE_FLOW_FIELD_TAG)
4870                         return rte_flow_error_set(error, ENOTSUP,
4871                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4872                                         "inner header fields modification"
4873                                         " is not supported");
4874         }
4875         if ((action_modify_field->dst.field ==
4876              action_modify_field->src.field) &&
4877             (action_modify_field->dst.level ==
4878              action_modify_field->src.level))
4879                 return rte_flow_error_set(error, EINVAL,
4880                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4881                                 "source and destination fields"
4882                                 " cannot be the same");
4883         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE ||
4884             action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER)
4885                 return rte_flow_error_set(error, EINVAL,
4886                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4887                                 "immediate value or a pointer to it"
4888                                 " cannot be used as a destination");
4889         if (action_modify_field->dst.field == RTE_FLOW_FIELD_START ||
4890             action_modify_field->src.field == RTE_FLOW_FIELD_START)
4891                 return rte_flow_error_set(error, ENOTSUP,
4892                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4893                                 "modifications of an arbitrary"
4894                                 " place in a packet is not supported");
4895         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE ||
4896             action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE)
4897                 return rte_flow_error_set(error, ENOTSUP,
4898                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4899                                 "modifications of the 802.1Q Tag"
4900                                 " Identifier is not supported");
4901         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI ||
4902             action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI)
4903                 return rte_flow_error_set(error, ENOTSUP,
4904                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4905                                 "modifications of the VXLAN Network"
4906                                 " Identifier is not supported");
4907         if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI ||
4908             action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI)
4909                 return rte_flow_error_set(error, ENOTSUP,
4910                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4911                                 "modifications of the GENEVE Network"
4912                                 " Identifier is not supported");
4913         if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK ||
4914             action_modify_field->src.field == RTE_FLOW_FIELD_MARK ||
4915             action_modify_field->dst.field == RTE_FLOW_FIELD_META ||
4916             action_modify_field->src.field == RTE_FLOW_FIELD_META) {
4917                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4918                     !mlx5_flow_ext_mreg_supported(dev))
4919                         return rte_flow_error_set(error, ENOTSUP,
4920                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4921                                         "cannot modify mark or metadata without"
4922                                         " extended metadata register support");
4923         }
4924         if (action_modify_field->operation != RTE_FLOW_MODIFY_SET)
4925                 return rte_flow_error_set(error, ENOTSUP,
4926                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4927                                 "add and sub operations"
4928                                 " are not supported");
4929         return (action_modify_field->width / 32) +
4930                !!(action_modify_field->width % 32);
4931 }
4932
4933 /**
4934  * Validate jump action.
4935  *
4936  * @param[in] action
4937  *   Pointer to the jump action.
4938  * @param[in] action_flags
4939  *   Holds the actions detected until now.
4940  * @param[in] attributes
4941  *   Pointer to flow attributes
4942  * @param[in] external
4943  *   Action belongs to flow rule created by request external to PMD.
4944  * @param[out] error
4945  *   Pointer to error structure.
4946  *
4947  * @return
4948  *   0 on success, a negative errno value otherwise and rte_errno is set.
4949  */
4950 static int
4951 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
4952                              const struct mlx5_flow_tunnel *tunnel,
4953                              const struct rte_flow_action *action,
4954                              uint64_t action_flags,
4955                              const struct rte_flow_attr *attributes,
4956                              bool external, struct rte_flow_error *error)
4957 {
4958         uint32_t target_group, table;
4959         int ret = 0;
4960         struct flow_grp_info grp_info = {
4961                 .external = !!external,
4962                 .transfer = !!attributes->transfer,
4963                 .fdb_def_rule = 1,
4964                 .std_tbl_fix = 0
4965         };
4966         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4967                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4968                 return rte_flow_error_set(error, EINVAL,
4969                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4970                                           "can't have 2 fate actions in"
4971                                           " same flow");
4972         if (!action->conf)
4973                 return rte_flow_error_set(error, EINVAL,
4974                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4975                                           NULL, "action configuration not set");
4976         target_group =
4977                 ((const struct rte_flow_action_jump *)action->conf)->group;
4978         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
4979                                        &grp_info, error);
4980         if (ret)
4981                 return ret;
4982         if (attributes->group == target_group &&
4983             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
4984                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
4985                 return rte_flow_error_set(error, EINVAL,
4986                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4987                                           "target group must be other than"
4988                                           " the current flow group");
4989         return 0;
4990 }
4991
4992 /*
4993  * Validate the port_id action.
4994  *
4995  * @param[in] dev
4996  *   Pointer to rte_eth_dev structure.
4997  * @param[in] action_flags
4998  *   Bit-fields that holds the actions detected until now.
4999  * @param[in] action
5000  *   Port_id RTE action structure.
5001  * @param[in] attr
5002  *   Attributes of flow that includes this action.
5003  * @param[out] error
5004  *   Pointer to error structure.
5005  *
5006  * @return
5007  *   0 on success, a negative errno value otherwise and rte_errno is set.
5008  */
5009 static int
5010 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
5011                                 uint64_t action_flags,
5012                                 const struct rte_flow_action *action,
5013                                 const struct rte_flow_attr *attr,
5014                                 struct rte_flow_error *error)
5015 {
5016         const struct rte_flow_action_port_id *port_id;
5017         struct mlx5_priv *act_priv;
5018         struct mlx5_priv *dev_priv;
5019         uint16_t port;
5020
5021         if (!attr->transfer)
5022                 return rte_flow_error_set(error, ENOTSUP,
5023                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5024                                           NULL,
5025                                           "port id action is valid in transfer"
5026                                           " mode only");
5027         if (!action || !action->conf)
5028                 return rte_flow_error_set(error, ENOTSUP,
5029                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5030                                           NULL,
5031                                           "port id action parameters must be"
5032                                           " specified");
5033         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
5034                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5035                 return rte_flow_error_set(error, EINVAL,
5036                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5037                                           "can have only one fate actions in"
5038                                           " a flow");
5039         dev_priv = mlx5_dev_to_eswitch_info(dev);
5040         if (!dev_priv)
5041                 return rte_flow_error_set(error, rte_errno,
5042                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5043                                           NULL,
5044                                           "failed to obtain E-Switch info");
5045         port_id = action->conf;
5046         port = port_id->original ? dev->data->port_id : port_id->id;
5047         act_priv = mlx5_port_to_eswitch_info(port, false);
5048         if (!act_priv)
5049                 return rte_flow_error_set
5050                                 (error, rte_errno,
5051                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
5052                                  "failed to obtain E-Switch port id for port");
5053         if (act_priv->domain_id != dev_priv->domain_id)
5054                 return rte_flow_error_set
5055                                 (error, EINVAL,
5056                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5057                                  "port does not belong to"
5058                                  " E-Switch being configured");
5059         return 0;
5060 }
5061
5062 /**
5063  * Get the maximum number of modify header actions.
5064  *
5065  * @param dev
5066  *   Pointer to rte_eth_dev structure.
5067  * @param root
5068  *   Whether action is on root table.
5069  *
5070  * @return
5071  *   Max number of modify header actions device can support.
5072  */
5073 static inline unsigned int
5074 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
5075                               bool root)
5076 {
5077         /*
5078          * There's no way to directly query the max capacity from FW.
5079          * The maximal value on root table should be assumed to be supported.
5080          */
5081         if (!root)
5082                 return MLX5_MAX_MODIFY_NUM;
5083         else
5084                 return MLX5_ROOT_TBL_MODIFY_NUM;
5085 }
5086
5087 /**
5088  * Validate the meter action.
5089  *
5090  * @param[in] dev
5091  *   Pointer to rte_eth_dev structure.
5092  * @param[in] action_flags
5093  *   Bit-fields that holds the actions detected until now.
5094  * @param[in] action
5095  *   Pointer to the meter action.
5096  * @param[in] attr
5097  *   Attributes of flow that includes this action.
5098  * @param[in] port_id_item
5099  *   Pointer to item indicating port id.
5100  * @param[out] error
5101  *   Pointer to error structure.
5102  *
5103  * @return
5104  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5105  */
5106 static int
5107 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
5108                                 uint64_t action_flags,
5109                                 const struct rte_flow_action *action,
5110                                 const struct rte_flow_attr *attr,
5111                                 const struct rte_flow_item *port_id_item,
5112                                 bool *def_policy,
5113                                 struct rte_flow_error *error)
5114 {
5115         struct mlx5_priv *priv = dev->data->dev_private;
5116         const struct rte_flow_action_meter *am = action->conf;
5117         struct mlx5_flow_meter_info *fm;
5118         struct mlx5_flow_meter_policy *mtr_policy;
5119         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
5120
5121         if (!am)
5122                 return rte_flow_error_set(error, EINVAL,
5123                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5124                                           "meter action conf is NULL");
5125
5126         if (action_flags & MLX5_FLOW_ACTION_METER)
5127                 return rte_flow_error_set(error, ENOTSUP,
5128                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5129                                           "meter chaining not support");
5130         if (action_flags & MLX5_FLOW_ACTION_JUMP)
5131                 return rte_flow_error_set(error, ENOTSUP,
5132                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5133                                           "meter with jump not support");
5134         if (!priv->mtr_en)
5135                 return rte_flow_error_set(error, ENOTSUP,
5136                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5137                                           NULL,
5138                                           "meter action not supported");
5139         fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL);
5140         if (!fm)
5141                 return rte_flow_error_set(error, EINVAL,
5142                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5143                                           "Meter not found");
5144         /* aso meter can always be shared by different domains */
5145         if (fm->ref_cnt && !priv->sh->meter_aso_en &&
5146             !(fm->transfer == attr->transfer ||
5147               (!fm->ingress && !attr->ingress && attr->egress) ||
5148               (!fm->egress && !attr->egress && attr->ingress)))
5149                 return rte_flow_error_set(error, EINVAL,
5150                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5151                         "Flow attributes domain are either invalid "
5152                         "or have a domain conflict with current "
5153                         "meter attributes");
5154         if (fm->def_policy) {
5155                 if (!((attr->transfer &&
5156                         mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) ||
5157                         (attr->egress &&
5158                         mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) ||
5159                         (attr->ingress &&
5160                         mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS])))
5161                         return rte_flow_error_set(error, EINVAL,
5162                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5163                                           "Flow attributes domain "
5164                                           "have a conflict with current "
5165                                           "meter domain attributes");
5166                 *def_policy = true;
5167         } else {
5168                 mtr_policy = mlx5_flow_meter_policy_find(dev,
5169                                                 fm->policy_id, NULL);
5170                 if (!mtr_policy)
5171                         return rte_flow_error_set(error, EINVAL,
5172                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5173                                           "Invalid policy id for meter ");
5174                 if (!((attr->transfer && mtr_policy->transfer) ||
5175                         (attr->egress && mtr_policy->egress) ||
5176                         (attr->ingress && mtr_policy->ingress)))
5177                         return rte_flow_error_set(error, EINVAL,
5178                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5179                                           "Flow attributes domain "
5180                                           "have a conflict with current "
5181                                           "meter domain attributes");
5182                 if (attr->transfer && mtr_policy->dev) {
5183                         /**
5184                          * When policy has fate action of port_id,
5185                          * the flow should have the same src port as policy.
5186                          */
5187                         struct mlx5_priv *policy_port_priv =
5188                                         mtr_policy->dev->data->dev_private;
5189                         int32_t flow_src_port = priv->representor_id;
5190
5191                         if (port_id_item) {
5192                                 const struct rte_flow_item_port_id *spec =
5193                                                         port_id_item->spec;
5194                                 struct mlx5_priv *port_priv =
5195                                         mlx5_port_to_eswitch_info(spec->id,
5196                                                                   false);
5197                                 if (!port_priv)
5198                                         return rte_flow_error_set(error,
5199                                                 rte_errno,
5200                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5201                                                 spec,
5202                                                 "Failed to get port info.");
5203                                 flow_src_port = port_priv->representor_id;
5204                         }
5205                         if (flow_src_port != policy_port_priv->representor_id)
5206                                 return rte_flow_error_set(error,
5207                                                 rte_errno,
5208                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5209                                                 NULL,
5210                                                 "Flow and meter policy "
5211                                                 "have different src port.");
5212                 }
5213                 *def_policy = false;
5214         }
5215         return 0;
5216 }
5217
5218 /**
5219  * Validate the age action.
5220  *
5221  * @param[in] action_flags
5222  *   Holds the actions detected until now.
5223  * @param[in] action
5224  *   Pointer to the age action.
5225  * @param[in] dev
5226  *   Pointer to the Ethernet device structure.
5227  * @param[out] error
5228  *   Pointer to error structure.
5229  *
5230  * @return
5231  *   0 on success, a negative errno value otherwise and rte_errno is set.
5232  */
5233 static int
5234 flow_dv_validate_action_age(uint64_t action_flags,
5235                             const struct rte_flow_action *action,
5236                             struct rte_eth_dev *dev,
5237                             struct rte_flow_error *error)
5238 {
5239         struct mlx5_priv *priv = dev->data->dev_private;
5240         const struct rte_flow_action_age *age = action->conf;
5241
5242         if (!priv->config.devx || (priv->sh->cmng.counter_fallback &&
5243             !priv->sh->aso_age_mng))
5244                 return rte_flow_error_set(error, ENOTSUP,
5245                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5246                                           NULL,
5247                                           "age action not supported");
5248         if (!(action->conf))
5249                 return rte_flow_error_set(error, EINVAL,
5250                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5251                                           "configuration cannot be null");
5252         if (!(age->timeout))
5253                 return rte_flow_error_set(error, EINVAL,
5254                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5255                                           "invalid timeout value 0");
5256         if (action_flags & MLX5_FLOW_ACTION_AGE)
5257                 return rte_flow_error_set(error, EINVAL,
5258                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5259                                           "duplicate age actions set");
5260         return 0;
5261 }
5262
5263 /**
5264  * Validate the modify-header IPv4 DSCP actions.
5265  *
5266  * @param[in] action_flags
5267  *   Holds the actions detected until now.
5268  * @param[in] action
5269  *   Pointer to the modify action.
5270  * @param[in] item_flags
5271  *   Holds the items detected.
5272  * @param[out] error
5273  *   Pointer to error structure.
5274  *
5275  * @return
5276  *   0 on success, a negative errno value otherwise and rte_errno is set.
5277  */
5278 static int
5279 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
5280                                          const struct rte_flow_action *action,
5281                                          const uint64_t item_flags,
5282                                          struct rte_flow_error *error)
5283 {
5284         int ret = 0;
5285
5286         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5287         if (!ret) {
5288                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
5289                         return rte_flow_error_set(error, EINVAL,
5290                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5291                                                   NULL,
5292                                                   "no ipv4 item in pattern");
5293         }
5294         return ret;
5295 }
5296
5297 /**
5298  * Validate the modify-header IPv6 DSCP actions.
5299  *
5300  * @param[in] action_flags
5301  *   Holds the actions detected until now.
5302  * @param[in] action
5303  *   Pointer to the modify action.
5304  * @param[in] item_flags
5305  *   Holds the items detected.
5306  * @param[out] error
5307  *   Pointer to error structure.
5308  *
5309  * @return
5310  *   0 on success, a negative errno value otherwise and rte_errno is set.
5311  */
5312 static int
5313 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
5314                                          const struct rte_flow_action *action,
5315                                          const uint64_t item_flags,
5316                                          struct rte_flow_error *error)
5317 {
5318         int ret = 0;
5319
5320         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5321         if (!ret) {
5322                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
5323                         return rte_flow_error_set(error, EINVAL,
5324                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5325                                                   NULL,
5326                                                   "no ipv6 item in pattern");
5327         }
5328         return ret;
5329 }
5330
5331 /**
5332  * Match modify-header resource.
5333  *
5334  * @param list
5335  *   Pointer to the hash list.
5336  * @param entry
5337  *   Pointer to exist resource entry object.
5338  * @param key
5339  *   Key of the new entry.
5340  * @param ctx
5341  *   Pointer to new modify-header resource.
5342  *
5343  * @return
5344  *   0 on matching, non-zero otherwise.
5345  */
5346 int
5347 flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
5348                         struct mlx5_hlist_entry *entry,
5349                         uint64_t key __rte_unused, void *cb_ctx)
5350 {
5351         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5352         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5353         struct mlx5_flow_dv_modify_hdr_resource *resource =
5354                         container_of(entry, typeof(*resource), entry);
5355         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5356
5357         key_len += ref->actions_num * sizeof(ref->actions[0]);
5358         return ref->actions_num != resource->actions_num ||
5359                memcmp(&ref->ft_type, &resource->ft_type, key_len);
5360 }
5361
5362 struct mlx5_hlist_entry *
5363 flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
5364                          void *cb_ctx)
5365 {
5366         struct mlx5_dev_ctx_shared *sh = list->ctx;
5367         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5368         struct mlx5dv_dr_domain *ns;
5369         struct mlx5_flow_dv_modify_hdr_resource *entry;
5370         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5371         int ret;
5372         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5373         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5374
5375         entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0,
5376                             SOCKET_ID_ANY);
5377         if (!entry) {
5378                 rte_flow_error_set(ctx->error, ENOMEM,
5379                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5380                                    "cannot allocate resource memory");
5381                 return NULL;
5382         }
5383         rte_memcpy(&entry->ft_type,
5384                    RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
5385                    key_len + data_len);
5386         if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
5387                 ns = sh->fdb_domain;
5388         else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
5389                 ns = sh->tx_domain;
5390         else
5391                 ns = sh->rx_domain;
5392         ret = mlx5_flow_os_create_flow_action_modify_header
5393                                         (sh->ctx, ns, entry,
5394                                          data_len, &entry->action);
5395         if (ret) {
5396                 mlx5_free(entry);
5397                 rte_flow_error_set(ctx->error, ENOMEM,
5398                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5399                                    NULL, "cannot create modification action");
5400                 return NULL;
5401         }
5402         return &entry->entry;
5403 }
5404
5405 /**
5406  * Validate the sample action.
5407  *
5408  * @param[in, out] action_flags
5409  *   Holds the actions detected until now.
5410  * @param[in] action
5411  *   Pointer to the sample action.
5412  * @param[in] dev
5413  *   Pointer to the Ethernet device structure.
5414  * @param[in] attr
5415  *   Attributes of flow that includes this action.
5416  * @param[in] item_flags
5417  *   Holds the items detected.
5418  * @param[in] rss
5419  *   Pointer to the RSS action.
5420  * @param[out] sample_rss
5421  *   Pointer to the RSS action in sample action list.
5422  * @param[out] count
5423  *   Pointer to the COUNT action in sample action list.
5424  * @param[out] fdb_mirror_limit
5425  *   Pointer to the FDB mirror limitation flag.
5426  * @param[out] error
5427  *   Pointer to error structure.
5428  *
5429  * @return
5430  *   0 on success, a negative errno value otherwise and rte_errno is set.
5431  */
5432 static int
5433 flow_dv_validate_action_sample(uint64_t *action_flags,
5434                                const struct rte_flow_action *action,
5435                                struct rte_eth_dev *dev,
5436                                const struct rte_flow_attr *attr,
5437                                uint64_t item_flags,
5438                                const struct rte_flow_action_rss *rss,
5439                                const struct rte_flow_action_rss **sample_rss,
5440                                const struct rte_flow_action_count **count,
5441                                int *fdb_mirror_limit,
5442                                struct rte_flow_error *error)
5443 {
5444         struct mlx5_priv *priv = dev->data->dev_private;
5445         struct mlx5_dev_config *dev_conf = &priv->config;
5446         const struct rte_flow_action_sample *sample = action->conf;
5447         const struct rte_flow_action *act;
5448         uint64_t sub_action_flags = 0;
5449         uint16_t queue_index = 0xFFFF;
5450         int actions_n = 0;
5451         int ret;
5452
5453         if (!sample)
5454                 return rte_flow_error_set(error, EINVAL,
5455                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5456                                           "configuration cannot be NULL");
5457         if (sample->ratio == 0)
5458                 return rte_flow_error_set(error, EINVAL,
5459                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5460                                           "ratio value starts from 1");
5461         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
5462                 return rte_flow_error_set(error, ENOTSUP,
5463                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5464                                           NULL,
5465                                           "sample action not supported");
5466         if (*action_flags & MLX5_FLOW_ACTION_SAMPLE)
5467                 return rte_flow_error_set(error, EINVAL,
5468                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5469                                           "Multiple sample actions not "
5470                                           "supported");
5471         if (*action_flags & MLX5_FLOW_ACTION_METER)
5472                 return rte_flow_error_set(error, EINVAL,
5473                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5474                                           "wrong action order, meter should "
5475                                           "be after sample action");
5476         if (*action_flags & MLX5_FLOW_ACTION_JUMP)
5477                 return rte_flow_error_set(error, EINVAL,
5478                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5479                                           "wrong action order, jump should "
5480                                           "be after sample action");
5481         act = sample->actions;
5482         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
5483                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5484                         return rte_flow_error_set(error, ENOTSUP,
5485                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5486                                                   act, "too many actions");
5487                 switch (act->type) {
5488                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5489                         ret = mlx5_flow_validate_action_queue(act,
5490                                                               sub_action_flags,
5491                                                               dev,
5492                                                               attr, error);
5493                         if (ret < 0)
5494                                 return ret;
5495                         queue_index = ((const struct rte_flow_action_queue *)
5496                                                         (act->conf))->index;
5497                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
5498                         ++actions_n;
5499                         break;
5500                 case RTE_FLOW_ACTION_TYPE_RSS:
5501                         *sample_rss = act->conf;
5502                         ret = mlx5_flow_validate_action_rss(act,
5503                                                             sub_action_flags,
5504                                                             dev, attr,
5505                                                             item_flags,
5506                                                             error);
5507                         if (ret < 0)
5508                                 return ret;
5509                         if (rss && *sample_rss &&
5510                             ((*sample_rss)->level != rss->level ||
5511                             (*sample_rss)->types != rss->types))
5512                                 return rte_flow_error_set(error, ENOTSUP,
5513                                         RTE_FLOW_ERROR_TYPE_ACTION,
5514                                         NULL,
5515                                         "Can't use the different RSS types "
5516                                         "or level in the same flow");
5517                         if (*sample_rss != NULL && (*sample_rss)->queue_num)
5518                                 queue_index = (*sample_rss)->queue[0];
5519                         sub_action_flags |= MLX5_FLOW_ACTION_RSS;
5520                         ++actions_n;
5521                         break;
5522                 case RTE_FLOW_ACTION_TYPE_MARK:
5523                         ret = flow_dv_validate_action_mark(dev, act,
5524                                                            sub_action_flags,
5525                                                            attr, error);
5526                         if (ret < 0)
5527                                 return ret;
5528                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
5529                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
5530                                                 MLX5_FLOW_ACTION_MARK_EXT;
5531                         else
5532                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
5533                         ++actions_n;
5534                         break;
5535                 case RTE_FLOW_ACTION_TYPE_COUNT:
5536                         ret = flow_dv_validate_action_count
5537                                 (dev, is_shared_action_count(act),
5538                                  *action_flags | sub_action_flags,
5539                                  error);
5540                         if (ret < 0)
5541                                 return ret;
5542                         *count = act->conf;
5543                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
5544                         *action_flags |= MLX5_FLOW_ACTION_COUNT;
5545                         ++actions_n;
5546                         break;
5547                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5548                         ret = flow_dv_validate_action_port_id(dev,
5549                                                               sub_action_flags,
5550                                                               act,
5551                                                               attr,
5552                                                               error);
5553                         if (ret)
5554                                 return ret;
5555                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5556                         ++actions_n;
5557                         break;
5558                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5559                         ret = flow_dv_validate_action_raw_encap_decap
5560                                 (dev, NULL, act->conf, attr, &sub_action_flags,
5561                                  &actions_n, action, item_flags, error);
5562                         if (ret < 0)
5563                                 return ret;
5564                         ++actions_n;
5565                         break;
5566                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5567                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5568                         ret = flow_dv_validate_action_l2_encap(dev,
5569                                                                sub_action_flags,
5570                                                                act, attr,
5571                                                                error);
5572                         if (ret < 0)
5573                                 return ret;
5574                         sub_action_flags |= MLX5_FLOW_ACTION_ENCAP;
5575                         ++actions_n;
5576                         break;
5577                 default:
5578                         return rte_flow_error_set(error, ENOTSUP,
5579                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5580                                                   NULL,
5581                                                   "Doesn't support optional "
5582                                                   "action");
5583                 }
5584         }
5585         if (attr->ingress && !attr->transfer) {
5586                 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE |
5587                                           MLX5_FLOW_ACTION_RSS)))
5588                         return rte_flow_error_set(error, EINVAL,
5589                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5590                                                   NULL,
5591                                                   "Ingress must has a dest "
5592                                                   "QUEUE for Sample");
5593         } else if (attr->egress && !attr->transfer) {
5594                 return rte_flow_error_set(error, ENOTSUP,
5595                                           RTE_FLOW_ERROR_TYPE_ACTION,
5596                                           NULL,
5597                                           "Sample Only support Ingress "
5598                                           "or E-Switch");
5599         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
5600                 MLX5_ASSERT(attr->transfer);
5601                 if (sample->ratio > 1)
5602                         return rte_flow_error_set(error, ENOTSUP,
5603                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5604                                                   NULL,
5605                                                   "E-Switch doesn't support "
5606                                                   "any optional action "
5607                                                   "for sampling");
5608                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
5609                         return rte_flow_error_set(error, ENOTSUP,
5610                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5611                                                   NULL,
5612                                                   "unsupported action QUEUE");
5613                 if (sub_action_flags & MLX5_FLOW_ACTION_RSS)
5614                         return rte_flow_error_set(error, ENOTSUP,
5615                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5616                                                   NULL,
5617                                                   "unsupported action QUEUE");
5618                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
5619                         return rte_flow_error_set(error, EINVAL,
5620                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5621                                                   NULL,
5622                                                   "E-Switch must has a dest "
5623                                                   "port for mirroring");
5624                 if (!priv->config.hca_attr.reg_c_preserve &&
5625                      priv->representor_id != UINT16_MAX)
5626                         *fdb_mirror_limit = 1;
5627         }
5628         /* Continue validation for Xcap actions.*/
5629         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
5630             (queue_index == 0xFFFF ||
5631              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5632                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5633                      MLX5_FLOW_XCAP_ACTIONS)
5634                         return rte_flow_error_set(error, ENOTSUP,
5635                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5636                                                   NULL, "encap and decap "
5637                                                   "combination aren't "
5638                                                   "supported");
5639                 if (!attr->transfer && attr->ingress && (sub_action_flags &
5640                                                         MLX5_FLOW_ACTION_ENCAP))
5641                         return rte_flow_error_set(error, ENOTSUP,
5642                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5643                                                   NULL, "encap is not supported"
5644                                                   " for ingress traffic");
5645         }
5646         return 0;
5647 }
5648
5649 /**
5650  * Find existing modify-header resource or create and register a new one.
5651  *
5652  * @param dev[in, out]
5653  *   Pointer to rte_eth_dev structure.
5654  * @param[in, out] resource
5655  *   Pointer to modify-header resource.
5656  * @parm[in, out] dev_flow
5657  *   Pointer to the dev_flow.
5658  * @param[out] error
5659  *   pointer to error structure.
5660  *
5661  * @return
5662  *   0 on success otherwise -errno and errno is set.
5663  */
5664 static int
5665 flow_dv_modify_hdr_resource_register
5666                         (struct rte_eth_dev *dev,
5667                          struct mlx5_flow_dv_modify_hdr_resource *resource,
5668                          struct mlx5_flow *dev_flow,
5669                          struct rte_flow_error *error)
5670 {
5671         struct mlx5_priv *priv = dev->data->dev_private;
5672         struct mlx5_dev_ctx_shared *sh = priv->sh;
5673         uint32_t key_len = sizeof(*resource) -
5674                            offsetof(typeof(*resource), ft_type) +
5675                            resource->actions_num * sizeof(resource->actions[0]);
5676         struct mlx5_hlist_entry *entry;
5677         struct mlx5_flow_cb_ctx ctx = {
5678                 .error = error,
5679                 .data = resource,
5680         };
5681         uint64_t key64;
5682
5683         resource->root = !dev_flow->dv.group;
5684         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
5685                                                                 resource->root))
5686                 return rte_flow_error_set(error, EOVERFLOW,
5687                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5688                                           "too many modify header items");
5689         key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
5690         entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
5691         if (!entry)
5692                 return -rte_errno;
5693         resource = container_of(entry, typeof(*resource), entry);
5694         dev_flow->handle->dvh.modify_hdr = resource;
5695         return 0;
5696 }
5697
5698 /**
5699  * Get DV flow counter by index.
5700  *
5701  * @param[in] dev
5702  *   Pointer to the Ethernet device structure.
5703  * @param[in] idx
5704  *   mlx5 flow counter index in the container.
5705  * @param[out] ppool
5706  *   mlx5 flow counter pool in the container.
5707  *
5708  * @return
5709  *   Pointer to the counter, NULL otherwise.
5710  */
5711 static struct mlx5_flow_counter *
5712 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
5713                            uint32_t idx,
5714                            struct mlx5_flow_counter_pool **ppool)
5715 {
5716         struct mlx5_priv *priv = dev->data->dev_private;
5717         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5718         struct mlx5_flow_counter_pool *pool;
5719
5720         /* Decrease to original index and clear shared bit. */
5721         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
5722         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
5723         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
5724         MLX5_ASSERT(pool);
5725         if (ppool)
5726                 *ppool = pool;
5727         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
5728 }
5729
5730 /**
5731  * Check the devx counter belongs to the pool.
5732  *
5733  * @param[in] pool
5734  *   Pointer to the counter pool.
5735  * @param[in] id
5736  *   The counter devx ID.
5737  *
5738  * @return
5739  *   True if counter belongs to the pool, false otherwise.
5740  */
5741 static bool
5742 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
5743 {
5744         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
5745                    MLX5_COUNTERS_PER_POOL;
5746
5747         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
5748                 return true;
5749         return false;
5750 }
5751
5752 /**
5753  * Get a pool by devx counter ID.
5754  *
5755  * @param[in] cmng
5756  *   Pointer to the counter management.
5757  * @param[in] id
5758  *   The counter devx ID.
5759  *
5760  * @return
5761  *   The counter pool pointer if exists, NULL otherwise,
5762  */
5763 static struct mlx5_flow_counter_pool *
5764 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
5765 {
5766         uint32_t i;
5767         struct mlx5_flow_counter_pool *pool = NULL;
5768
5769         rte_spinlock_lock(&cmng->pool_update_sl);
5770         /* Check last used pool. */
5771         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
5772             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
5773                 pool = cmng->pools[cmng->last_pool_idx];
5774                 goto out;
5775         }
5776         /* ID out of range means no suitable pool in the container. */
5777         if (id > cmng->max_id || id < cmng->min_id)
5778                 goto out;
5779         /*
5780          * Find the pool from the end of the container, since mostly counter
5781          * ID is sequence increasing, and the last pool should be the needed
5782          * one.
5783          */
5784         i = cmng->n_valid;
5785         while (i--) {
5786                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
5787
5788                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
5789                         pool = pool_tmp;
5790                         break;
5791                 }
5792         }
5793 out:
5794         rte_spinlock_unlock(&cmng->pool_update_sl);
5795         return pool;
5796 }
5797
5798 /**
5799  * Resize a counter container.
5800  *
5801  * @param[in] dev
5802  *   Pointer to the Ethernet device structure.
5803  *
5804  * @return
5805  *   0 on success, otherwise negative errno value and rte_errno is set.
5806  */
5807 static int
5808 flow_dv_container_resize(struct rte_eth_dev *dev)
5809 {
5810         struct mlx5_priv *priv = dev->data->dev_private;
5811         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5812         void *old_pools = cmng->pools;
5813         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
5814         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
5815         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
5816
5817         if (!pools) {
5818                 rte_errno = ENOMEM;
5819                 return -ENOMEM;
5820         }
5821         if (old_pools)
5822                 memcpy(pools, old_pools, cmng->n *
5823                                        sizeof(struct mlx5_flow_counter_pool *));
5824         cmng->n = resize;
5825         cmng->pools = pools;
5826         if (old_pools)
5827                 mlx5_free(old_pools);
5828         return 0;
5829 }
5830
5831 /**
5832  * Query a devx flow counter.
5833  *
5834  * @param[in] dev
5835  *   Pointer to the Ethernet device structure.
5836  * @param[in] counter
5837  *   Index to the flow counter.
5838  * @param[out] pkts
5839  *   The statistics value of packets.
5840  * @param[out] bytes
5841  *   The statistics value of bytes.
5842  *
5843  * @return
5844  *   0 on success, otherwise a negative errno value and rte_errno is set.
5845  */
5846 static inline int
5847 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
5848                      uint64_t *bytes)
5849 {
5850         struct mlx5_priv *priv = dev->data->dev_private;
5851         struct mlx5_flow_counter_pool *pool = NULL;
5852         struct mlx5_flow_counter *cnt;
5853         int offset;
5854
5855         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5856         MLX5_ASSERT(pool);
5857         if (priv->sh->cmng.counter_fallback)
5858                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
5859                                         0, pkts, bytes, 0, NULL, NULL, 0);
5860         rte_spinlock_lock(&pool->sl);
5861         if (!pool->raw) {
5862                 *pkts = 0;
5863                 *bytes = 0;
5864         } else {
5865                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
5866                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
5867                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
5868         }
5869         rte_spinlock_unlock(&pool->sl);
5870         return 0;
5871 }
5872
5873 /**
5874  * Create and initialize a new counter pool.
5875  *
5876  * @param[in] dev
5877  *   Pointer to the Ethernet device structure.
5878  * @param[out] dcs
5879  *   The devX counter handle.
5880  * @param[in] age
5881  *   Whether the pool is for counter that was allocated for aging.
5882  * @param[in/out] cont_cur
5883  *   Pointer to the container pointer, it will be update in pool resize.
5884  *
5885  * @return
5886  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
5887  */
5888 static struct mlx5_flow_counter_pool *
5889 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
5890                     uint32_t age)
5891 {
5892         struct mlx5_priv *priv = dev->data->dev_private;
5893         struct mlx5_flow_counter_pool *pool;
5894         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5895         bool fallback = priv->sh->cmng.counter_fallback;
5896         uint32_t size = sizeof(*pool);
5897
5898         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
5899         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
5900         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
5901         if (!pool) {
5902                 rte_errno = ENOMEM;
5903                 return NULL;
5904         }
5905         pool->raw = NULL;
5906         pool->is_aged = !!age;
5907         pool->query_gen = 0;
5908         pool->min_dcs = dcs;
5909         rte_spinlock_init(&pool->sl);
5910         rte_spinlock_init(&pool->csl);
5911         TAILQ_INIT(&pool->counters[0]);
5912         TAILQ_INIT(&pool->counters[1]);
5913         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
5914         rte_spinlock_lock(&cmng->pool_update_sl);
5915         pool->index = cmng->n_valid;
5916         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
5917                 mlx5_free(pool);
5918                 rte_spinlock_unlock(&cmng->pool_update_sl);
5919                 return NULL;
5920         }
5921         cmng->pools[pool->index] = pool;
5922         cmng->n_valid++;
5923         if (unlikely(fallback)) {
5924                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
5925
5926                 if (base < cmng->min_id)
5927                         cmng->min_id = base;
5928                 if (base > cmng->max_id)
5929                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
5930                 cmng->last_pool_idx = pool->index;
5931         }
5932         rte_spinlock_unlock(&cmng->pool_update_sl);
5933         return pool;
5934 }
5935
5936 /**
5937  * Prepare a new counter and/or a new counter pool.
5938  *
5939  * @param[in] dev
5940  *   Pointer to the Ethernet device structure.
5941  * @param[out] cnt_free
5942  *   Where to put the pointer of a new counter.
5943  * @param[in] age
5944  *   Whether the pool is for counter that was allocated for aging.
5945  *
5946  * @return
5947  *   The counter pool pointer and @p cnt_free is set on success,
5948  *   NULL otherwise and rte_errno is set.
5949  */
5950 static struct mlx5_flow_counter_pool *
5951 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
5952                              struct mlx5_flow_counter **cnt_free,
5953                              uint32_t age)
5954 {
5955         struct mlx5_priv *priv = dev->data->dev_private;
5956         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5957         struct mlx5_flow_counter_pool *pool;
5958         struct mlx5_counters tmp_tq;
5959         struct mlx5_devx_obj *dcs = NULL;
5960         struct mlx5_flow_counter *cnt;
5961         enum mlx5_counter_type cnt_type =
5962                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
5963         bool fallback = priv->sh->cmng.counter_fallback;
5964         uint32_t i;
5965
5966         if (fallback) {
5967                 /* bulk_bitmap must be 0 for single counter allocation. */
5968                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
5969                 if (!dcs)
5970                         return NULL;
5971                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
5972                 if (!pool) {
5973                         pool = flow_dv_pool_create(dev, dcs, age);
5974                         if (!pool) {
5975                                 mlx5_devx_cmd_destroy(dcs);
5976                                 return NULL;
5977                         }
5978                 }
5979                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
5980                 cnt = MLX5_POOL_GET_CNT(pool, i);
5981                 cnt->pool = pool;
5982                 cnt->dcs_when_free = dcs;
5983                 *cnt_free = cnt;
5984                 return pool;
5985         }
5986         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
5987         if (!dcs) {
5988                 rte_errno = ENODATA;
5989                 return NULL;
5990         }
5991         pool = flow_dv_pool_create(dev, dcs, age);
5992         if (!pool) {
5993                 mlx5_devx_cmd_destroy(dcs);
5994                 return NULL;
5995         }
5996         TAILQ_INIT(&tmp_tq);
5997         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
5998                 cnt = MLX5_POOL_GET_CNT(pool, i);
5999                 cnt->pool = pool;
6000                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
6001         }
6002         rte_spinlock_lock(&cmng->csl[cnt_type]);
6003         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
6004         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6005         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
6006         (*cnt_free)->pool = pool;
6007         return pool;
6008 }
6009
6010 /**
6011  * Allocate a flow counter.
6012  *
6013  * @param[in] dev
6014  *   Pointer to the Ethernet device structure.
6015  * @param[in] age
6016  *   Whether the counter was allocated for aging.
6017  *
6018  * @return
6019  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6020  */
6021 static uint32_t
6022 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
6023 {
6024         struct mlx5_priv *priv = dev->data->dev_private;
6025         struct mlx5_flow_counter_pool *pool = NULL;
6026         struct mlx5_flow_counter *cnt_free = NULL;
6027         bool fallback = priv->sh->cmng.counter_fallback;
6028         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6029         enum mlx5_counter_type cnt_type =
6030                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6031         uint32_t cnt_idx;
6032
6033         if (!priv->config.devx) {
6034                 rte_errno = ENOTSUP;
6035                 return 0;
6036         }
6037         /* Get free counters from container. */
6038         rte_spinlock_lock(&cmng->csl[cnt_type]);
6039         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
6040         if (cnt_free)
6041                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
6042         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6043         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
6044                 goto err;
6045         pool = cnt_free->pool;
6046         if (fallback)
6047                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
6048         /* Create a DV counter action only in the first time usage. */
6049         if (!cnt_free->action) {
6050                 uint16_t offset;
6051                 struct mlx5_devx_obj *dcs;
6052                 int ret;
6053
6054                 if (!fallback) {
6055                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
6056                         dcs = pool->min_dcs;
6057                 } else {
6058                         offset = 0;
6059                         dcs = cnt_free->dcs_when_free;
6060                 }
6061                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
6062                                                             &cnt_free->action);
6063                 if (ret) {
6064                         rte_errno = errno;
6065                         goto err;
6066                 }
6067         }
6068         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
6069                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
6070         /* Update the counter reset values. */
6071         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
6072                                  &cnt_free->bytes))
6073                 goto err;
6074         if (!fallback && !priv->sh->cmng.query_thread_on)
6075                 /* Start the asynchronous batch query by the host thread. */
6076                 mlx5_set_query_alarm(priv->sh);
6077         /*
6078          * When the count action isn't shared (by ID), shared_info field is
6079          * used for indirect action API's refcnt.
6080          * When the counter action is not shared neither by ID nor by indirect
6081          * action API, shared info must be 1.
6082          */
6083         cnt_free->shared_info.refcnt = 1;
6084         return cnt_idx;
6085 err:
6086         if (cnt_free) {
6087                 cnt_free->pool = pool;
6088                 if (fallback)
6089                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
6090                 rte_spinlock_lock(&cmng->csl[cnt_type]);
6091                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
6092                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
6093         }
6094         return 0;
6095 }
6096
6097 /**
6098  * Allocate a shared flow counter.
6099  *
6100  * @param[in] ctx
6101  *   Pointer to the shared counter configuration.
6102  * @param[in] data
6103  *   Pointer to save the allocated counter index.
6104  *
6105  * @return
6106  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6107  */
6108
6109 static int32_t
6110 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
6111 {
6112         struct mlx5_shared_counter_conf *conf = ctx;
6113         struct rte_eth_dev *dev = conf->dev;
6114         struct mlx5_flow_counter *cnt;
6115
6116         data->dword = flow_dv_counter_alloc(dev, 0);
6117         data->dword |= MLX5_CNT_SHARED_OFFSET;
6118         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
6119         cnt->shared_info.id = conf->id;
6120         return 0;
6121 }
6122
6123 /**
6124  * Get a shared flow counter.
6125  *
6126  * @param[in] dev
6127  *   Pointer to the Ethernet device structure.
6128  * @param[in] id
6129  *   Counter identifier.
6130  *
6131  * @return
6132  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6133  */
6134 static uint32_t
6135 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
6136 {
6137         struct mlx5_priv *priv = dev->data->dev_private;
6138         struct mlx5_shared_counter_conf conf = {
6139                 .dev = dev,
6140                 .id = id,
6141         };
6142         union mlx5_l3t_data data = {
6143                 .dword = 0,
6144         };
6145
6146         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
6147                                flow_dv_counter_alloc_shared_cb, &conf);
6148         return data.dword;
6149 }
6150
6151 /**
6152  * Get age param from counter index.
6153  *
6154  * @param[in] dev
6155  *   Pointer to the Ethernet device structure.
6156  * @param[in] counter
6157  *   Index to the counter handler.
6158  *
6159  * @return
6160  *   The aging parameter specified for the counter index.
6161  */
6162 static struct mlx5_age_param*
6163 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
6164                                 uint32_t counter)
6165 {
6166         struct mlx5_flow_counter *cnt;
6167         struct mlx5_flow_counter_pool *pool = NULL;
6168
6169         flow_dv_counter_get_by_idx(dev, counter, &pool);
6170         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
6171         cnt = MLX5_POOL_GET_CNT(pool, counter);
6172         return MLX5_CNT_TO_AGE(cnt);
6173 }
6174
6175 /**
6176  * Remove a flow counter from aged counter list.
6177  *
6178  * @param[in] dev
6179  *   Pointer to the Ethernet device structure.
6180  * @param[in] counter
6181  *   Index to the counter handler.
6182  * @param[in] cnt
6183  *   Pointer to the counter handler.
6184  */
6185 static void
6186 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
6187                                 uint32_t counter, struct mlx5_flow_counter *cnt)
6188 {
6189         struct mlx5_age_info *age_info;
6190         struct mlx5_age_param *age_param;
6191         struct mlx5_priv *priv = dev->data->dev_private;
6192         uint16_t expected = AGE_CANDIDATE;
6193
6194         age_info = GET_PORT_AGE_INFO(priv);
6195         age_param = flow_dv_counter_idx_get_age(dev, counter);
6196         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
6197                                          AGE_FREE, false, __ATOMIC_RELAXED,
6198                                          __ATOMIC_RELAXED)) {
6199                 /**
6200                  * We need the lock even it is age timeout,
6201                  * since counter may still in process.
6202                  */
6203                 rte_spinlock_lock(&age_info->aged_sl);
6204                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
6205                 rte_spinlock_unlock(&age_info->aged_sl);
6206                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
6207         }
6208 }
6209
6210 /**
6211  * Release a flow counter.
6212  *
6213  * @param[in] dev
6214  *   Pointer to the Ethernet device structure.
6215  * @param[in] counter
6216  *   Index to the counter handler.
6217  */
6218 static void
6219 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter)
6220 {
6221         struct mlx5_priv *priv = dev->data->dev_private;
6222         struct mlx5_flow_counter_pool *pool = NULL;
6223         struct mlx5_flow_counter *cnt;
6224         enum mlx5_counter_type cnt_type;
6225
6226         if (!counter)
6227                 return;
6228         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6229         MLX5_ASSERT(pool);
6230         if (pool->is_aged) {
6231                 flow_dv_counter_remove_from_age(dev, counter, cnt);
6232         } else {
6233                 /*
6234                  * If the counter action is shared by ID, the l3t_clear_entry
6235                  * function reduces its references counter. If after the
6236                  * reduction the action is still referenced, the function
6237                  * returns here and does not release it.
6238                  */
6239                 if (IS_LEGACY_SHARED_CNT(counter) &&
6240                     mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
6241                                          cnt->shared_info.id))
6242                         return;
6243                 /*
6244                  * If the counter action is shared by indirect action API,
6245                  * the atomic function reduces its references counter.
6246                  * If after the reduction the action is still referenced, the
6247                  * function returns here and does not release it.
6248                  * When the counter action is not shared neither by ID nor by
6249                  * indirect action API, shared info is 1 before the reduction,
6250                  * so this condition is failed and function doesn't return here.
6251                  */
6252                 if (!IS_LEGACY_SHARED_CNT(counter) &&
6253                     __atomic_sub_fetch(&cnt->shared_info.refcnt, 1,
6254                                        __ATOMIC_RELAXED))
6255                         return;
6256         }
6257         cnt->pool = pool;
6258         /*
6259          * Put the counter back to list to be updated in none fallback mode.
6260          * Currently, we are using two list alternately, while one is in query,
6261          * add the freed counter to the other list based on the pool query_gen
6262          * value. After query finishes, add counter the list to the global
6263          * container counter list. The list changes while query starts. In
6264          * this case, lock will not be needed as query callback and release
6265          * function both operate with the different list.
6266          */
6267         if (!priv->sh->cmng.counter_fallback) {
6268                 rte_spinlock_lock(&pool->csl);
6269                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
6270                 rte_spinlock_unlock(&pool->csl);
6271         } else {
6272                 cnt->dcs_when_free = cnt->dcs_when_active;
6273                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6274                                            MLX5_COUNTER_TYPE_ORIGIN;
6275                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
6276                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
6277                                   cnt, next);
6278                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
6279         }
6280 }
6281
6282 /**
6283  * Resize a meter id container.
6284  *
6285  * @param[in] dev
6286  *   Pointer to the Ethernet device structure.
6287  *
6288  * @return
6289  *   0 on success, otherwise negative errno value and rte_errno is set.
6290  */
6291 static int
6292 flow_dv_mtr_container_resize(struct rte_eth_dev *dev)
6293 {
6294         struct mlx5_priv *priv = dev->data->dev_private;
6295         struct mlx5_aso_mtr_pools_mng *pools_mng =
6296                                 &priv->sh->mtrmng->pools_mng;
6297         void *old_pools = pools_mng->pools;
6298         uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE;
6299         uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize;
6300         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
6301
6302         if (!pools) {
6303                 rte_errno = ENOMEM;
6304                 return -ENOMEM;
6305         }
6306         if (!pools_mng->n)
6307                 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) {
6308                         mlx5_free(pools);
6309                         return -ENOMEM;
6310                 }
6311         if (old_pools)
6312                 memcpy(pools, old_pools, pools_mng->n *
6313                                        sizeof(struct mlx5_aso_mtr_pool *));
6314         pools_mng->n = resize;
6315         pools_mng->pools = pools;
6316         if (old_pools)
6317                 mlx5_free(old_pools);
6318         return 0;
6319 }
6320
6321 /**
6322  * Prepare a new meter and/or a new meter pool.
6323  *
6324  * @param[in] dev
6325  *   Pointer to the Ethernet device structure.
6326  * @param[out] mtr_free
6327  *   Where to put the pointer of a new meter.g.
6328  *
6329  * @return
6330  *   The meter pool pointer and @mtr_free is set on success,
6331  *   NULL otherwise and rte_errno is set.
6332  */
6333 static struct mlx5_aso_mtr_pool *
6334 flow_dv_mtr_pool_create(struct rte_eth_dev *dev,
6335                              struct mlx5_aso_mtr **mtr_free)
6336 {
6337         struct mlx5_priv *priv = dev->data->dev_private;
6338         struct mlx5_aso_mtr_pools_mng *pools_mng =
6339                                 &priv->sh->mtrmng->pools_mng;
6340         struct mlx5_aso_mtr_pool *pool = NULL;
6341         struct mlx5_devx_obj *dcs = NULL;
6342         uint32_t i;
6343         uint32_t log_obj_size;
6344
6345         log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
6346         dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx,
6347                         priv->sh->pdn, log_obj_size);
6348         if (!dcs) {
6349                 rte_errno = ENODATA;
6350                 return NULL;
6351         }
6352         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
6353         if (!pool) {
6354                 rte_errno = ENOMEM;
6355                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6356                 return NULL;
6357         }
6358         pool->devx_obj = dcs;
6359         pool->index = pools_mng->n_valid;
6360         if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) {
6361                 mlx5_free(pool);
6362                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6363                 return NULL;
6364         }
6365         pools_mng->pools[pool->index] = pool;
6366         pools_mng->n_valid++;
6367         for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) {
6368                 pool->mtrs[i].offset = i;
6369                 LIST_INSERT_HEAD(&pools_mng->meters,
6370                                                 &pool->mtrs[i], next);
6371         }
6372         pool->mtrs[0].offset = 0;
6373         *mtr_free = &pool->mtrs[0];
6374         return pool;
6375 }
6376
6377 /**
6378  * Release a flow meter into pool.
6379  *
6380  * @param[in] dev
6381  *   Pointer to the Ethernet device structure.
6382  * @param[in] mtr_idx
6383  *   Index to aso flow meter.
6384  */
6385 static void
6386 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx)
6387 {
6388         struct mlx5_priv *priv = dev->data->dev_private;
6389         struct mlx5_aso_mtr_pools_mng *pools_mng =
6390                                 &priv->sh->mtrmng->pools_mng;
6391         struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
6392
6393         MLX5_ASSERT(aso_mtr);
6394         rte_spinlock_lock(&pools_mng->mtrsl);
6395         memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info));
6396         aso_mtr->state = ASO_METER_FREE;
6397         LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next);
6398         rte_spinlock_unlock(&pools_mng->mtrsl);
6399 }
6400
6401 /**
6402  * Allocate a aso flow meter.
6403  *
6404  * @param[in] dev
6405  *   Pointer to the Ethernet device structure.
6406  *
6407  * @return
6408  *   Index to aso flow meter on success, 0 otherwise and rte_errno is set.
6409  */
6410 static uint32_t
6411 flow_dv_mtr_alloc(struct rte_eth_dev *dev)
6412 {
6413         struct mlx5_priv *priv = dev->data->dev_private;
6414         struct mlx5_aso_mtr *mtr_free = NULL;
6415         struct mlx5_aso_mtr_pools_mng *pools_mng =
6416                                 &priv->sh->mtrmng->pools_mng;
6417         struct mlx5_aso_mtr_pool *pool;
6418         uint32_t mtr_idx = 0;
6419
6420         if (!priv->config.devx) {
6421                 rte_errno = ENOTSUP;
6422                 return 0;
6423         }
6424         /* Allocate the flow meter memory. */
6425         /* Get free meters from management. */
6426         rte_spinlock_lock(&pools_mng->mtrsl);
6427         mtr_free = LIST_FIRST(&pools_mng->meters);
6428         if (mtr_free)
6429                 LIST_REMOVE(mtr_free, next);
6430         if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) {
6431                 rte_spinlock_unlock(&pools_mng->mtrsl);
6432                 return 0;
6433         }
6434         mtr_free->state = ASO_METER_WAIT;
6435         rte_spinlock_unlock(&pools_mng->mtrsl);
6436         pool = container_of(mtr_free,
6437                         struct mlx5_aso_mtr_pool,
6438                         mtrs[mtr_free->offset]);
6439         mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
6440         if (!mtr_free->fm.meter_action) {
6441 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
6442                 struct rte_flow_error error;
6443                 uint8_t reg_id;
6444
6445                 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
6446                 mtr_free->fm.meter_action =
6447                         mlx5_glue->dv_create_flow_action_aso
6448                                                 (priv->sh->rx_domain,
6449                                                  pool->devx_obj->obj,
6450                                                  mtr_free->offset,
6451                                                  (1 << MLX5_FLOW_COLOR_GREEN),
6452                                                  reg_id - REG_C_0);
6453 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
6454                 if (!mtr_free->fm.meter_action) {
6455                         flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
6456                         return 0;
6457                 }
6458         }
6459         return mtr_idx;
6460 }
6461
6462 /**
6463  * Verify the @p attributes will be correctly understood by the NIC and store
6464  * them in the @p flow if everything is correct.
6465  *
6466  * @param[in] dev
6467  *   Pointer to dev struct.
6468  * @param[in] attributes
6469  *   Pointer to flow attributes
6470  * @param[in] external
6471  *   This flow rule is created by request external to PMD.
6472  * @param[out] error
6473  *   Pointer to error structure.
6474  *
6475  * @return
6476  *   - 0 on success and non root table.
6477  *   - 1 on success and root table.
6478  *   - a negative errno value otherwise and rte_errno is set.
6479  */
6480 static int
6481 flow_dv_validate_attributes(struct rte_eth_dev *dev,
6482                             const struct mlx5_flow_tunnel *tunnel,
6483                             const struct rte_flow_attr *attributes,
6484                             const struct flow_grp_info *grp_info,
6485                             struct rte_flow_error *error)
6486 {
6487         struct mlx5_priv *priv = dev->data->dev_private;
6488         uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes);
6489         int ret = 0;
6490
6491 #ifndef HAVE_MLX5DV_DR
6492         RTE_SET_USED(tunnel);
6493         RTE_SET_USED(grp_info);
6494         if (attributes->group)
6495                 return rte_flow_error_set(error, ENOTSUP,
6496                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6497                                           NULL,
6498                                           "groups are not supported");
6499 #else
6500         uint32_t table = 0;
6501
6502         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
6503                                        grp_info, error);
6504         if (ret)
6505                 return ret;
6506         if (!table)
6507                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
6508 #endif
6509         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
6510             attributes->priority > lowest_priority)
6511                 return rte_flow_error_set(error, ENOTSUP,
6512                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
6513                                           NULL,
6514                                           "priority out of range");
6515         if (attributes->transfer) {
6516                 if (!priv->config.dv_esw_en)
6517                         return rte_flow_error_set
6518                                 (error, ENOTSUP,
6519                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6520                                  "E-Switch dr is not supported");
6521                 if (!(priv->representor || priv->master))
6522                         return rte_flow_error_set
6523                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6524                                  NULL, "E-Switch configuration can only be"
6525                                  " done by a master or a representor device");
6526                 if (attributes->egress)
6527                         return rte_flow_error_set
6528                                 (error, ENOTSUP,
6529                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
6530                                  "egress is not supported");
6531         }
6532         if (!(attributes->egress ^ attributes->ingress))
6533                 return rte_flow_error_set(error, ENOTSUP,
6534                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
6535                                           "must specify exactly one of "
6536                                           "ingress or egress");
6537         return ret;
6538 }
6539
6540 static uint16_t
6541 mlx5_flow_locate_proto_l3(const struct rte_flow_item **head,
6542                           const struct rte_flow_item *end)
6543 {
6544         const struct rte_flow_item *item = *head;
6545         uint16_t l3_protocol;
6546
6547         for (; item != end; item++) {
6548                 switch (item->type) {
6549                 default:
6550                         break;
6551                 case RTE_FLOW_ITEM_TYPE_IPV4:
6552                         l3_protocol = RTE_ETHER_TYPE_IPV4;
6553                         goto l3_ok;
6554                 case RTE_FLOW_ITEM_TYPE_IPV6:
6555                         l3_protocol = RTE_ETHER_TYPE_IPV6;
6556                         goto l3_ok;
6557                 case RTE_FLOW_ITEM_TYPE_ETH:
6558                         if (item->mask && item->spec) {
6559                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_eth,
6560                                                             type, item,
6561                                                             l3_protocol);
6562                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6563                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6564                                         goto l3_ok;
6565                         }
6566                         break;
6567                 case RTE_FLOW_ITEM_TYPE_VLAN:
6568                         if (item->mask && item->spec) {
6569                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_vlan,
6570                                                             inner_type, item,
6571                                                             l3_protocol);
6572                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6573                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6574                                         goto l3_ok;
6575                         }
6576                         break;
6577                 }
6578         }
6579         return 0;
6580 l3_ok:
6581         *head = item;
6582         return l3_protocol;
6583 }
6584
6585 static uint8_t
6586 mlx5_flow_locate_proto_l4(const struct rte_flow_item **head,
6587                           const struct rte_flow_item *end)
6588 {
6589         const struct rte_flow_item *item = *head;
6590         uint8_t l4_protocol;
6591
6592         for (; item != end; item++) {
6593                 switch (item->type) {
6594                 default:
6595                         break;
6596                 case RTE_FLOW_ITEM_TYPE_TCP:
6597                         l4_protocol = IPPROTO_TCP;
6598                         goto l4_ok;
6599                 case RTE_FLOW_ITEM_TYPE_UDP:
6600                         l4_protocol = IPPROTO_UDP;
6601                         goto l4_ok;
6602                 case RTE_FLOW_ITEM_TYPE_IPV4:
6603                         if (item->mask && item->spec) {
6604                                 const struct rte_flow_item_ipv4 *mask, *spec;
6605
6606                                 mask = (typeof(mask))item->mask;
6607                                 spec = (typeof(spec))item->spec;
6608                                 l4_protocol = mask->hdr.next_proto_id &
6609                                               spec->hdr.next_proto_id;
6610                                 if (l4_protocol == IPPROTO_TCP ||
6611                                     l4_protocol == IPPROTO_UDP)
6612                                         goto l4_ok;
6613                         }
6614                         break;
6615                 case RTE_FLOW_ITEM_TYPE_IPV6:
6616                         if (item->mask && item->spec) {
6617                                 const struct rte_flow_item_ipv6 *mask, *spec;
6618                                 mask = (typeof(mask))item->mask;
6619                                 spec = (typeof(spec))item->spec;
6620                                 l4_protocol = mask->hdr.proto & spec->hdr.proto;
6621                                 if (l4_protocol == IPPROTO_TCP ||
6622                                     l4_protocol == IPPROTO_UDP)
6623                                         goto l4_ok;
6624                         }
6625                         break;
6626                 }
6627         }
6628         return 0;
6629 l4_ok:
6630         *head = item;
6631         return l4_protocol;
6632 }
6633
6634 static int
6635 flow_dv_validate_item_integrity(struct rte_eth_dev *dev,
6636                                 const struct rte_flow_item *rule_items,
6637                                 const struct rte_flow_item *integrity_item,
6638                                 struct rte_flow_error *error)
6639 {
6640         struct mlx5_priv *priv = dev->data->dev_private;
6641         const struct rte_flow_item *tunnel_item, *end_item, *item = rule_items;
6642         const struct rte_flow_item_integrity *mask = (typeof(mask))
6643                                                      integrity_item->mask;
6644         const struct rte_flow_item_integrity *spec = (typeof(spec))
6645                                                      integrity_item->spec;
6646         uint32_t protocol;
6647
6648         if (!priv->config.hca_attr.pkt_integrity_match)
6649                 return rte_flow_error_set(error, ENOTSUP,
6650                                           RTE_FLOW_ERROR_TYPE_ITEM,
6651                                           integrity_item,
6652                                           "packet integrity integrity_item not supported");
6653         if (!mask)
6654                 mask = &rte_flow_item_integrity_mask;
6655         if (!mlx5_validate_integrity_item(mask))
6656                 return rte_flow_error_set(error, ENOTSUP,
6657                                           RTE_FLOW_ERROR_TYPE_ITEM,
6658                                           integrity_item,
6659                                           "unsupported integrity filter");
6660         tunnel_item = mlx5_flow_find_tunnel_item(rule_items);
6661         if (spec->level > 1) {
6662                 if (!tunnel_item)
6663                         return rte_flow_error_set(error, ENOTSUP,
6664                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6665                                                   integrity_item,
6666                                                   "missing tunnel item");
6667                 item = tunnel_item;
6668                 end_item = mlx5_find_end_item(tunnel_item);
6669         } else {
6670                 end_item = tunnel_item ? tunnel_item :
6671                            mlx5_find_end_item(integrity_item);
6672         }
6673         if (mask->l3_ok || mask->ipv4_csum_ok) {
6674                 protocol = mlx5_flow_locate_proto_l3(&item, end_item);
6675                 if (!protocol)
6676                         return rte_flow_error_set(error, EINVAL,
6677                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6678                                                   integrity_item,
6679                                                   "missing L3 protocol");
6680         }
6681         if (mask->l4_ok || mask->l4_csum_ok) {
6682                 protocol = mlx5_flow_locate_proto_l4(&item, end_item);
6683                 if (!protocol)
6684                         return rte_flow_error_set(error, EINVAL,
6685                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6686                                                   integrity_item,
6687                                                   "missing L4 protocol");
6688         }
6689         return 0;
6690 }
6691
6692 /**
6693  * Internal validation function. For validating both actions and items.
6694  *
6695  * @param[in] dev
6696  *   Pointer to the rte_eth_dev structure.
6697  * @param[in] attr
6698  *   Pointer to the flow attributes.
6699  * @param[in] items
6700  *   Pointer to the list of items.
6701  * @param[in] actions
6702  *   Pointer to the list of actions.
6703  * @param[in] external
6704  *   This flow rule is created by request external to PMD.
6705  * @param[in] hairpin
6706  *   Number of hairpin TX actions, 0 means classic flow.
6707  * @param[out] error
6708  *   Pointer to the error structure.
6709  *
6710  * @return
6711  *   0 on success, a negative errno value otherwise and rte_errno is set.
6712  */
6713 static int
6714 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
6715                  const struct rte_flow_item items[],
6716                  const struct rte_flow_action actions[],
6717                  bool external, int hairpin, struct rte_flow_error *error)
6718 {
6719         int ret;
6720         uint64_t action_flags = 0;
6721         uint64_t item_flags = 0;
6722         uint64_t last_item = 0;
6723         uint8_t next_protocol = 0xff;
6724         uint16_t ether_type = 0;
6725         int actions_n = 0;
6726         uint8_t item_ipv6_proto = 0;
6727         int fdb_mirror_limit = 0;
6728         int modify_after_mirror = 0;
6729         const struct rte_flow_item *geneve_item = NULL;
6730         const struct rte_flow_item *gre_item = NULL;
6731         const struct rte_flow_item *gtp_item = NULL;
6732         const struct rte_flow_action_raw_decap *decap;
6733         const struct rte_flow_action_raw_encap *encap;
6734         const struct rte_flow_action_rss *rss = NULL;
6735         const struct rte_flow_action_rss *sample_rss = NULL;
6736         const struct rte_flow_action_count *sample_count = NULL;
6737         const struct rte_flow_item_tcp nic_tcp_mask = {
6738                 .hdr = {
6739                         .tcp_flags = 0xFF,
6740                         .src_port = RTE_BE16(UINT16_MAX),
6741                         .dst_port = RTE_BE16(UINT16_MAX),
6742                 }
6743         };
6744         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
6745                 .hdr = {
6746                         .src_addr =
6747                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6748                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6749                         .dst_addr =
6750                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6751                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6752                         .vtc_flow = RTE_BE32(0xffffffff),
6753                         .proto = 0xff,
6754                         .hop_limits = 0xff,
6755                 },
6756                 .has_frag_ext = 1,
6757         };
6758         const struct rte_flow_item_ecpri nic_ecpri_mask = {
6759                 .hdr = {
6760                         .common = {
6761                                 .u32 =
6762                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
6763                                         .type = 0xFF,
6764                                         }).u32),
6765                         },
6766                         .dummy[0] = 0xffffffff,
6767                 },
6768         };
6769         struct mlx5_priv *priv = dev->data->dev_private;
6770         struct mlx5_dev_config *dev_conf = &priv->config;
6771         uint16_t queue_index = 0xFFFF;
6772         const struct rte_flow_item_vlan *vlan_m = NULL;
6773         uint32_t rw_act_num = 0;
6774         uint64_t is_root;
6775         const struct mlx5_flow_tunnel *tunnel;
6776         enum mlx5_tof_rule_type tof_rule_type;
6777         struct flow_grp_info grp_info = {
6778                 .external = !!external,
6779                 .transfer = !!attr->transfer,
6780                 .fdb_def_rule = !!priv->fdb_def_rule,
6781                 .std_tbl_fix = true,
6782         };
6783         const struct rte_eth_hairpin_conf *conf;
6784         const struct rte_flow_item *rule_items = items;
6785         const struct rte_flow_item *port_id_item = NULL;
6786         bool def_policy = false;
6787
6788         if (items == NULL)
6789                 return -1;
6790         tunnel = is_tunnel_offload_active(dev) ?
6791                  mlx5_get_tof(items, actions, &tof_rule_type) : NULL;
6792         if (tunnel) {
6793                 if (priv->representor)
6794                         return rte_flow_error_set
6795                                 (error, ENOTSUP,
6796                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6797                                  NULL, "decap not supported for VF representor");
6798                 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE)
6799                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6800                 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE)
6801                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
6802                                         MLX5_FLOW_ACTION_DECAP;
6803                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
6804                                         (dev, attr, tunnel, tof_rule_type);
6805         }
6806         ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error);
6807         if (ret < 0)
6808                 return ret;
6809         is_root = (uint64_t)ret;
6810         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6811                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6812                 int type = items->type;
6813
6814                 if (!mlx5_flow_os_item_supported(type))
6815                         return rte_flow_error_set(error, ENOTSUP,
6816                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6817                                                   NULL, "item not supported");
6818                 switch (type) {
6819                 case RTE_FLOW_ITEM_TYPE_VOID:
6820                         break;
6821                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6822                         ret = flow_dv_validate_item_port_id
6823                                         (dev, items, attr, item_flags, error);
6824                         if (ret < 0)
6825                                 return ret;
6826                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6827                         port_id_item = items;
6828                         break;
6829                 case RTE_FLOW_ITEM_TYPE_ETH:
6830                         ret = mlx5_flow_validate_item_eth(items, item_flags,
6831                                                           true, error);
6832                         if (ret < 0)
6833                                 return ret;
6834                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
6835                                              MLX5_FLOW_LAYER_OUTER_L2;
6836                         if (items->mask != NULL && items->spec != NULL) {
6837                                 ether_type =
6838                                         ((const struct rte_flow_item_eth *)
6839                                          items->spec)->type;
6840                                 ether_type &=
6841                                         ((const struct rte_flow_item_eth *)
6842                                          items->mask)->type;
6843                                 ether_type = rte_be_to_cpu_16(ether_type);
6844                         } else {
6845                                 ether_type = 0;
6846                         }
6847                         break;
6848                 case RTE_FLOW_ITEM_TYPE_VLAN:
6849                         ret = flow_dv_validate_item_vlan(items, item_flags,
6850                                                          dev, error);
6851                         if (ret < 0)
6852                                 return ret;
6853                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
6854                                              MLX5_FLOW_LAYER_OUTER_VLAN;
6855                         if (items->mask != NULL && items->spec != NULL) {
6856                                 ether_type =
6857                                         ((const struct rte_flow_item_vlan *)
6858                                          items->spec)->inner_type;
6859                                 ether_type &=
6860                                         ((const struct rte_flow_item_vlan *)
6861                                          items->mask)->inner_type;
6862                                 ether_type = rte_be_to_cpu_16(ether_type);
6863                         } else {
6864                                 ether_type = 0;
6865                         }
6866                         /* Store outer VLAN mask for of_push_vlan action. */
6867                         if (!tunnel)
6868                                 vlan_m = items->mask;
6869                         break;
6870                 case RTE_FLOW_ITEM_TYPE_IPV4:
6871                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6872                                                   &item_flags, &tunnel);
6873                         ret = flow_dv_validate_item_ipv4(items, item_flags,
6874                                                          last_item, ether_type,
6875                                                          error);
6876                         if (ret < 0)
6877                                 return ret;
6878                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
6879                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
6880                         if (items->mask != NULL &&
6881                             ((const struct rte_flow_item_ipv4 *)
6882                              items->mask)->hdr.next_proto_id) {
6883                                 next_protocol =
6884                                         ((const struct rte_flow_item_ipv4 *)
6885                                          (items->spec))->hdr.next_proto_id;
6886                                 next_protocol &=
6887                                         ((const struct rte_flow_item_ipv4 *)
6888                                          (items->mask))->hdr.next_proto_id;
6889                         } else {
6890                                 /* Reset for inner layer. */
6891                                 next_protocol = 0xff;
6892                         }
6893                         break;
6894                 case RTE_FLOW_ITEM_TYPE_IPV6:
6895                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6896                                                   &item_flags, &tunnel);
6897                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
6898                                                            last_item,
6899                                                            ether_type,
6900                                                            &nic_ipv6_mask,
6901                                                            error);
6902                         if (ret < 0)
6903                                 return ret;
6904                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
6905                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
6906                         if (items->mask != NULL &&
6907                             ((const struct rte_flow_item_ipv6 *)
6908                              items->mask)->hdr.proto) {
6909                                 item_ipv6_proto =
6910                                         ((const struct rte_flow_item_ipv6 *)
6911                                          items->spec)->hdr.proto;
6912                                 next_protocol =
6913                                         ((const struct rte_flow_item_ipv6 *)
6914                                          items->spec)->hdr.proto;
6915                                 next_protocol &=
6916                                         ((const struct rte_flow_item_ipv6 *)
6917                                          items->mask)->hdr.proto;
6918                         } else {
6919                                 /* Reset for inner layer. */
6920                                 next_protocol = 0xff;
6921                         }
6922                         break;
6923                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
6924                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
6925                                                                   item_flags,
6926                                                                   error);
6927                         if (ret < 0)
6928                                 return ret;
6929                         last_item = tunnel ?
6930                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
6931                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
6932                         if (items->mask != NULL &&
6933                             ((const struct rte_flow_item_ipv6_frag_ext *)
6934                              items->mask)->hdr.next_header) {
6935                                 next_protocol =
6936                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6937                                  items->spec)->hdr.next_header;
6938                                 next_protocol &=
6939                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6940                                  items->mask)->hdr.next_header;
6941                         } else {
6942                                 /* Reset for inner layer. */
6943                                 next_protocol = 0xff;
6944                         }
6945                         break;
6946                 case RTE_FLOW_ITEM_TYPE_TCP:
6947                         ret = mlx5_flow_validate_item_tcp
6948                                                 (items, item_flags,
6949                                                  next_protocol,
6950                                                  &nic_tcp_mask,
6951                                                  error);
6952                         if (ret < 0)
6953                                 return ret;
6954                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
6955                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
6956                         break;
6957                 case RTE_FLOW_ITEM_TYPE_UDP:
6958                         ret = mlx5_flow_validate_item_udp(items, item_flags,
6959                                                           next_protocol,
6960                                                           error);
6961                         if (ret < 0)
6962                                 return ret;
6963                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
6964                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
6965                         break;
6966                 case RTE_FLOW_ITEM_TYPE_GRE:
6967                         ret = mlx5_flow_validate_item_gre(items, item_flags,
6968                                                           next_protocol, error);
6969                         if (ret < 0)
6970                                 return ret;
6971                         gre_item = items;
6972                         last_item = MLX5_FLOW_LAYER_GRE;
6973                         break;
6974                 case RTE_FLOW_ITEM_TYPE_NVGRE:
6975                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
6976                                                             next_protocol,
6977                                                             error);
6978                         if (ret < 0)
6979                                 return ret;
6980                         last_item = MLX5_FLOW_LAYER_NVGRE;
6981                         break;
6982                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
6983                         ret = mlx5_flow_validate_item_gre_key
6984                                 (items, item_flags, gre_item, error);
6985                         if (ret < 0)
6986                                 return ret;
6987                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
6988                         break;
6989                 case RTE_FLOW_ITEM_TYPE_VXLAN:
6990                         ret = mlx5_flow_validate_item_vxlan(dev, items,
6991                                                             item_flags, attr,
6992                                                             error);
6993                         if (ret < 0)
6994                                 return ret;
6995                         last_item = MLX5_FLOW_LAYER_VXLAN;
6996                         break;
6997                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
6998                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
6999                                                                 item_flags, dev,
7000                                                                 error);
7001                         if (ret < 0)
7002                                 return ret;
7003                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
7004                         break;
7005                 case RTE_FLOW_ITEM_TYPE_GENEVE:
7006                         ret = mlx5_flow_validate_item_geneve(items,
7007                                                              item_flags, dev,
7008                                                              error);
7009                         if (ret < 0)
7010                                 return ret;
7011                         geneve_item = items;
7012                         last_item = MLX5_FLOW_LAYER_GENEVE;
7013                         break;
7014                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
7015                         ret = mlx5_flow_validate_item_geneve_opt(items,
7016                                                                  last_item,
7017                                                                  geneve_item,
7018                                                                  dev,
7019                                                                  error);
7020                         if (ret < 0)
7021                                 return ret;
7022                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
7023                         break;
7024                 case RTE_FLOW_ITEM_TYPE_MPLS:
7025                         ret = mlx5_flow_validate_item_mpls(dev, items,
7026                                                            item_flags,
7027                                                            last_item, error);
7028                         if (ret < 0)
7029                                 return ret;
7030                         last_item = MLX5_FLOW_LAYER_MPLS;
7031                         break;
7032
7033                 case RTE_FLOW_ITEM_TYPE_MARK:
7034                         ret = flow_dv_validate_item_mark(dev, items, attr,
7035                                                          error);
7036                         if (ret < 0)
7037                                 return ret;
7038                         last_item = MLX5_FLOW_ITEM_MARK;
7039                         break;
7040                 case RTE_FLOW_ITEM_TYPE_META:
7041                         ret = flow_dv_validate_item_meta(dev, items, attr,
7042                                                          error);
7043                         if (ret < 0)
7044                                 return ret;
7045                         last_item = MLX5_FLOW_ITEM_METADATA;
7046                         break;
7047                 case RTE_FLOW_ITEM_TYPE_ICMP:
7048                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
7049                                                            next_protocol,
7050                                                            error);
7051                         if (ret < 0)
7052                                 return ret;
7053                         last_item = MLX5_FLOW_LAYER_ICMP;
7054                         break;
7055                 case RTE_FLOW_ITEM_TYPE_ICMP6:
7056                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
7057                                                             next_protocol,
7058                                                             error);
7059                         if (ret < 0)
7060                                 return ret;
7061                         item_ipv6_proto = IPPROTO_ICMPV6;
7062                         last_item = MLX5_FLOW_LAYER_ICMP6;
7063                         break;
7064                 case RTE_FLOW_ITEM_TYPE_TAG:
7065                         ret = flow_dv_validate_item_tag(dev, items,
7066                                                         attr, error);
7067                         if (ret < 0)
7068                                 return ret;
7069                         last_item = MLX5_FLOW_ITEM_TAG;
7070                         break;
7071                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
7072                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
7073                         break;
7074                 case RTE_FLOW_ITEM_TYPE_GTP:
7075                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
7076                                                         error);
7077                         if (ret < 0)
7078                                 return ret;
7079                         gtp_item = items;
7080                         last_item = MLX5_FLOW_LAYER_GTP;
7081                         break;
7082                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
7083                         ret = flow_dv_validate_item_gtp_psc(items, last_item,
7084                                                             gtp_item, attr,
7085                                                             error);
7086                         if (ret < 0)
7087                                 return ret;
7088                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
7089                         break;
7090                 case RTE_FLOW_ITEM_TYPE_ECPRI:
7091                         /* Capacity will be checked in the translate stage. */
7092                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
7093                                                             last_item,
7094                                                             ether_type,
7095                                                             &nic_ecpri_mask,
7096                                                             error);
7097                         if (ret < 0)
7098                                 return ret;
7099                         last_item = MLX5_FLOW_LAYER_ECPRI;
7100                         break;
7101                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
7102                         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY)
7103                                 return rte_flow_error_set
7104                                         (error, ENOTSUP,
7105                                          RTE_FLOW_ERROR_TYPE_ITEM,
7106                                          NULL, "multiple integrity items not supported");
7107                         ret = flow_dv_validate_item_integrity(dev, rule_items,
7108                                                               items, error);
7109                         if (ret < 0)
7110                                 return ret;
7111                         last_item = MLX5_FLOW_ITEM_INTEGRITY;
7112                         break;
7113                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
7114                         ret = flow_dv_validate_item_aso_ct(dev, items,
7115                                                            &item_flags, error);
7116                         if (ret < 0)
7117                                 return ret;
7118                         break;
7119                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
7120                         /* tunnel offload item was processed before
7121                          * list it here as a supported type
7122                          */
7123                         break;
7124                 default:
7125                         return rte_flow_error_set(error, ENOTSUP,
7126                                                   RTE_FLOW_ERROR_TYPE_ITEM,
7127                                                   NULL, "item not supported");
7128                 }
7129                 item_flags |= last_item;
7130         }
7131         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7132                 int type = actions->type;
7133                 bool shared_count = false;
7134
7135                 if (!mlx5_flow_os_action_supported(type))
7136                         return rte_flow_error_set(error, ENOTSUP,
7137                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7138                                                   actions,
7139                                                   "action not supported");
7140                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
7141                         return rte_flow_error_set(error, ENOTSUP,
7142                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7143                                                   actions, "too many actions");
7144                 if (action_flags &
7145                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
7146                         return rte_flow_error_set(error, ENOTSUP,
7147                                 RTE_FLOW_ERROR_TYPE_ACTION,
7148                                 NULL, "meter action with policy "
7149                                 "must be the last action");
7150                 switch (type) {
7151                 case RTE_FLOW_ACTION_TYPE_VOID:
7152                         break;
7153                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7154                         ret = flow_dv_validate_action_port_id(dev,
7155                                                               action_flags,
7156                                                               actions,
7157                                                               attr,
7158                                                               error);
7159                         if (ret)
7160                                 return ret;
7161                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7162                         ++actions_n;
7163                         break;
7164                 case RTE_FLOW_ACTION_TYPE_FLAG:
7165                         ret = flow_dv_validate_action_flag(dev, action_flags,
7166                                                            attr, error);
7167                         if (ret < 0)
7168                                 return ret;
7169                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7170                                 /* Count all modify-header actions as one. */
7171                                 if (!(action_flags &
7172                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7173                                         ++actions_n;
7174                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
7175                                                 MLX5_FLOW_ACTION_MARK_EXT;
7176                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7177                                         modify_after_mirror = 1;
7178
7179                         } else {
7180                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
7181                                 ++actions_n;
7182                         }
7183                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7184                         break;
7185                 case RTE_FLOW_ACTION_TYPE_MARK:
7186                         ret = flow_dv_validate_action_mark(dev, actions,
7187                                                            action_flags,
7188                                                            attr, error);
7189                         if (ret < 0)
7190                                 return ret;
7191                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7192                                 /* Count all modify-header actions as one. */
7193                                 if (!(action_flags &
7194                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7195                                         ++actions_n;
7196                                 action_flags |= MLX5_FLOW_ACTION_MARK |
7197                                                 MLX5_FLOW_ACTION_MARK_EXT;
7198                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7199                                         modify_after_mirror = 1;
7200                         } else {
7201                                 action_flags |= MLX5_FLOW_ACTION_MARK;
7202                                 ++actions_n;
7203                         }
7204                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7205                         break;
7206                 case RTE_FLOW_ACTION_TYPE_SET_META:
7207                         ret = flow_dv_validate_action_set_meta(dev, actions,
7208                                                                action_flags,
7209                                                                attr, error);
7210                         if (ret < 0)
7211                                 return ret;
7212                         /* Count all modify-header actions as one action. */
7213                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7214                                 ++actions_n;
7215                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7216                                 modify_after_mirror = 1;
7217                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7218                         rw_act_num += MLX5_ACT_NUM_SET_META;
7219                         break;
7220                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7221                         ret = flow_dv_validate_action_set_tag(dev, actions,
7222                                                               action_flags,
7223                                                               attr, error);
7224                         if (ret < 0)
7225                                 return ret;
7226                         /* Count all modify-header actions as one action. */
7227                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7228                                 ++actions_n;
7229                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7230                                 modify_after_mirror = 1;
7231                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7232                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7233                         break;
7234                 case RTE_FLOW_ACTION_TYPE_DROP:
7235                         ret = mlx5_flow_validate_action_drop(action_flags,
7236                                                              attr, error);
7237                         if (ret < 0)
7238                                 return ret;
7239                         action_flags |= MLX5_FLOW_ACTION_DROP;
7240                         ++actions_n;
7241                         break;
7242                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7243                         ret = mlx5_flow_validate_action_queue(actions,
7244                                                               action_flags, dev,
7245                                                               attr, error);
7246                         if (ret < 0)
7247                                 return ret;
7248                         queue_index = ((const struct rte_flow_action_queue *)
7249                                                         (actions->conf))->index;
7250                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7251                         ++actions_n;
7252                         break;
7253                 case RTE_FLOW_ACTION_TYPE_RSS:
7254                         rss = actions->conf;
7255                         ret = mlx5_flow_validate_action_rss(actions,
7256                                                             action_flags, dev,
7257                                                             attr, item_flags,
7258                                                             error);
7259                         if (ret < 0)
7260                                 return ret;
7261                         if (rss && sample_rss &&
7262                             (sample_rss->level != rss->level ||
7263                             sample_rss->types != rss->types))
7264                                 return rte_flow_error_set(error, ENOTSUP,
7265                                         RTE_FLOW_ERROR_TYPE_ACTION,
7266                                         NULL,
7267                                         "Can't use the different RSS types "
7268                                         "or level in the same flow");
7269                         if (rss != NULL && rss->queue_num)
7270                                 queue_index = rss->queue[0];
7271                         action_flags |= MLX5_FLOW_ACTION_RSS;
7272                         ++actions_n;
7273                         break;
7274                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
7275                         ret =
7276                         mlx5_flow_validate_action_default_miss(action_flags,
7277                                         attr, error);
7278                         if (ret < 0)
7279                                 return ret;
7280                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
7281                         ++actions_n;
7282                         break;
7283                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
7284                 case RTE_FLOW_ACTION_TYPE_COUNT:
7285                         shared_count = is_shared_action_count(actions);
7286                         ret = flow_dv_validate_action_count(dev, shared_count,
7287                                                             action_flags,
7288                                                             error);
7289                         if (ret < 0)
7290                                 return ret;
7291                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7292                         ++actions_n;
7293                         break;
7294                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7295                         if (flow_dv_validate_action_pop_vlan(dev,
7296                                                              action_flags,
7297                                                              actions,
7298                                                              item_flags, attr,
7299                                                              error))
7300                                 return -rte_errno;
7301                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7302                                 modify_after_mirror = 1;
7303                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7304                         ++actions_n;
7305                         break;
7306                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
7307                         ret = flow_dv_validate_action_push_vlan(dev,
7308                                                                 action_flags,
7309                                                                 vlan_m,
7310                                                                 actions, attr,
7311                                                                 error);
7312                         if (ret < 0)
7313                                 return ret;
7314                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7315                                 modify_after_mirror = 1;
7316                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
7317                         ++actions_n;
7318                         break;
7319                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
7320                         ret = flow_dv_validate_action_set_vlan_pcp
7321                                                 (action_flags, actions, error);
7322                         if (ret < 0)
7323                                 return ret;
7324                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7325                                 modify_after_mirror = 1;
7326                         /* Count PCP with push_vlan command. */
7327                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
7328                         break;
7329                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
7330                         ret = flow_dv_validate_action_set_vlan_vid
7331                                                 (item_flags, action_flags,
7332                                                  actions, error);
7333                         if (ret < 0)
7334                                 return ret;
7335                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7336                                 modify_after_mirror = 1;
7337                         /* Count VID with push_vlan command. */
7338                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
7339                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
7340                         break;
7341                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
7342                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
7343                         ret = flow_dv_validate_action_l2_encap(dev,
7344                                                                action_flags,
7345                                                                actions, attr,
7346                                                                error);
7347                         if (ret < 0)
7348                                 return ret;
7349                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
7350                         ++actions_n;
7351                         break;
7352                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
7353                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
7354                         ret = flow_dv_validate_action_decap(dev, action_flags,
7355                                                             actions, item_flags,
7356                                                             attr, error);
7357                         if (ret < 0)
7358                                 return ret;
7359                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7360                                 modify_after_mirror = 1;
7361                         action_flags |= MLX5_FLOW_ACTION_DECAP;
7362                         ++actions_n;
7363                         break;
7364                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
7365                         ret = flow_dv_validate_action_raw_encap_decap
7366                                 (dev, NULL, actions->conf, attr, &action_flags,
7367                                  &actions_n, actions, item_flags, error);
7368                         if (ret < 0)
7369                                 return ret;
7370                         break;
7371                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
7372                         decap = actions->conf;
7373                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
7374                                 ;
7375                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
7376                                 encap = NULL;
7377                                 actions--;
7378                         } else {
7379                                 encap = actions->conf;
7380                         }
7381                         ret = flow_dv_validate_action_raw_encap_decap
7382                                            (dev,
7383                                             decap ? decap : &empty_decap, encap,
7384                                             attr, &action_flags, &actions_n,
7385                                             actions, item_flags, error);
7386                         if (ret < 0)
7387                                 return ret;
7388                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7389                             (action_flags & MLX5_FLOW_ACTION_DECAP))
7390                                 modify_after_mirror = 1;
7391                         break;
7392                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
7393                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
7394                         ret = flow_dv_validate_action_modify_mac(action_flags,
7395                                                                  actions,
7396                                                                  item_flags,
7397                                                                  error);
7398                         if (ret < 0)
7399                                 return ret;
7400                         /* Count all modify-header actions as one action. */
7401                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7402                                 ++actions_n;
7403                         action_flags |= actions->type ==
7404                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
7405                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
7406                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
7407                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7408                                 modify_after_mirror = 1;
7409                         /*
7410                          * Even if the source and destination MAC addresses have
7411                          * overlap in the header with 4B alignment, the convert
7412                          * function will handle them separately and 4 SW actions
7413                          * will be created. And 2 actions will be added each
7414                          * time no matter how many bytes of address will be set.
7415                          */
7416                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
7417                         break;
7418                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
7419                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
7420                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
7421                                                                   actions,
7422                                                                   item_flags,
7423                                                                   error);
7424                         if (ret < 0)
7425                                 return ret;
7426                         /* Count all modify-header actions as one action. */
7427                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7428                                 ++actions_n;
7429                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7430                                 modify_after_mirror = 1;
7431                         action_flags |= actions->type ==
7432                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
7433                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
7434                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
7435                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
7436                         break;
7437                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
7438                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
7439                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
7440                                                                   actions,
7441                                                                   item_flags,
7442                                                                   error);
7443                         if (ret < 0)
7444                                 return ret;
7445                         if (item_ipv6_proto == IPPROTO_ICMPV6)
7446                                 return rte_flow_error_set(error, ENOTSUP,
7447                                         RTE_FLOW_ERROR_TYPE_ACTION,
7448                                         actions,
7449                                         "Can't change header "
7450                                         "with ICMPv6 proto");
7451                         /* Count all modify-header actions as one action. */
7452                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7453                                 ++actions_n;
7454                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7455                                 modify_after_mirror = 1;
7456                         action_flags |= actions->type ==
7457                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
7458                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
7459                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
7460                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
7461                         break;
7462                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
7463                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
7464                         ret = flow_dv_validate_action_modify_tp(action_flags,
7465                                                                 actions,
7466                                                                 item_flags,
7467                                                                 error);
7468                         if (ret < 0)
7469                                 return ret;
7470                         /* Count all modify-header actions as one action. */
7471                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7472                                 ++actions_n;
7473                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7474                                 modify_after_mirror = 1;
7475                         action_flags |= actions->type ==
7476                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
7477                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
7478                                                 MLX5_FLOW_ACTION_SET_TP_DST;
7479                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
7480                         break;
7481                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
7482                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
7483                         ret = flow_dv_validate_action_modify_ttl(action_flags,
7484                                                                  actions,
7485                                                                  item_flags,
7486                                                                  error);
7487                         if (ret < 0)
7488                                 return ret;
7489                         /* Count all modify-header actions as one action. */
7490                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7491                                 ++actions_n;
7492                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7493                                 modify_after_mirror = 1;
7494                         action_flags |= actions->type ==
7495                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
7496                                                 MLX5_FLOW_ACTION_SET_TTL :
7497                                                 MLX5_FLOW_ACTION_DEC_TTL;
7498                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
7499                         break;
7500                 case RTE_FLOW_ACTION_TYPE_JUMP:
7501                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
7502                                                            action_flags,
7503                                                            attr, external,
7504                                                            error);
7505                         if (ret)
7506                                 return ret;
7507                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7508                             fdb_mirror_limit)
7509                                 return rte_flow_error_set(error, EINVAL,
7510                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7511                                                   NULL,
7512                                                   "sample and jump action combination is not supported");
7513                         ++actions_n;
7514                         action_flags |= MLX5_FLOW_ACTION_JUMP;
7515                         break;
7516                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
7517                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
7518                         ret = flow_dv_validate_action_modify_tcp_seq
7519                                                                 (action_flags,
7520                                                                  actions,
7521                                                                  item_flags,
7522                                                                  error);
7523                         if (ret < 0)
7524                                 return ret;
7525                         /* Count all modify-header actions as one action. */
7526                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7527                                 ++actions_n;
7528                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7529                                 modify_after_mirror = 1;
7530                         action_flags |= actions->type ==
7531                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
7532                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
7533                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
7534                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
7535                         break;
7536                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
7537                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
7538                         ret = flow_dv_validate_action_modify_tcp_ack
7539                                                                 (action_flags,
7540                                                                  actions,
7541                                                                  item_flags,
7542                                                                  error);
7543                         if (ret < 0)
7544                                 return ret;
7545                         /* Count all modify-header actions as one action. */
7546                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7547                                 ++actions_n;
7548                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7549                                 modify_after_mirror = 1;
7550                         action_flags |= actions->type ==
7551                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
7552                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
7553                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
7554                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
7555                         break;
7556                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7557                         break;
7558                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
7559                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7560                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7561                         break;
7562                 case RTE_FLOW_ACTION_TYPE_METER:
7563                         ret = mlx5_flow_validate_action_meter(dev,
7564                                                               action_flags,
7565                                                               actions, attr,
7566                                                               port_id_item,
7567                                                               &def_policy,
7568                                                               error);
7569                         if (ret < 0)
7570                                 return ret;
7571                         action_flags |= MLX5_FLOW_ACTION_METER;
7572                         if (!def_policy)
7573                                 action_flags |=
7574                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
7575                         ++actions_n;
7576                         /* Meter action will add one more TAG action. */
7577                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7578                         break;
7579                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
7580                         if (!attr->transfer && !attr->group)
7581                                 return rte_flow_error_set(error, ENOTSUP,
7582                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7583                                                                            NULL,
7584                           "Shared ASO age action is not supported for group 0");
7585                         if (action_flags & MLX5_FLOW_ACTION_AGE)
7586                                 return rte_flow_error_set
7587                                                   (error, EINVAL,
7588                                                    RTE_FLOW_ERROR_TYPE_ACTION,
7589                                                    NULL,
7590                                                    "duplicate age actions set");
7591                         action_flags |= MLX5_FLOW_ACTION_AGE;
7592                         ++actions_n;
7593                         break;
7594                 case RTE_FLOW_ACTION_TYPE_AGE:
7595                         ret = flow_dv_validate_action_age(action_flags,
7596                                                           actions, dev,
7597                                                           error);
7598                         if (ret < 0)
7599                                 return ret;
7600                         /*
7601                          * Validate the regular AGE action (using counter)
7602                          * mutual exclusion with share counter actions.
7603                          */
7604                         if (!priv->sh->flow_hit_aso_en) {
7605                                 if (shared_count)
7606                                         return rte_flow_error_set
7607                                                 (error, EINVAL,
7608                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7609                                                 NULL,
7610                                                 "old age and shared count combination is not supported");
7611                                 if (sample_count)
7612                                         return rte_flow_error_set
7613                                                 (error, EINVAL,
7614                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7615                                                 NULL,
7616                                                 "old age action and count must be in the same sub flow");
7617                         }
7618                         action_flags |= MLX5_FLOW_ACTION_AGE;
7619                         ++actions_n;
7620                         break;
7621                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
7622                         ret = flow_dv_validate_action_modify_ipv4_dscp
7623                                                          (action_flags,
7624                                                           actions,
7625                                                           item_flags,
7626                                                           error);
7627                         if (ret < 0)
7628                                 return ret;
7629                         /* Count all modify-header actions as one action. */
7630                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7631                                 ++actions_n;
7632                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7633                                 modify_after_mirror = 1;
7634                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
7635                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7636                         break;
7637                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
7638                         ret = flow_dv_validate_action_modify_ipv6_dscp
7639                                                                 (action_flags,
7640                                                                  actions,
7641                                                                  item_flags,
7642                                                                  error);
7643                         if (ret < 0)
7644                                 return ret;
7645                         /* Count all modify-header actions as one action. */
7646                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7647                                 ++actions_n;
7648                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7649                                 modify_after_mirror = 1;
7650                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
7651                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7652                         break;
7653                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
7654                         ret = flow_dv_validate_action_sample(&action_flags,
7655                                                              actions, dev,
7656                                                              attr, item_flags,
7657                                                              rss, &sample_rss,
7658                                                              &sample_count,
7659                                                              &fdb_mirror_limit,
7660                                                              error);
7661                         if (ret < 0)
7662                                 return ret;
7663                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
7664                         ++actions_n;
7665                         break;
7666                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
7667                         ret = flow_dv_validate_action_modify_field(dev,
7668                                                                    action_flags,
7669                                                                    actions,
7670                                                                    attr,
7671                                                                    error);
7672                         if (ret < 0)
7673                                 return ret;
7674                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7675                                 modify_after_mirror = 1;
7676                         /* Count all modify-header actions as one action. */
7677                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7678                                 ++actions_n;
7679                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
7680                         rw_act_num += ret;
7681                         break;
7682                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
7683                         ret = flow_dv_validate_action_aso_ct(dev, action_flags,
7684                                                              item_flags, attr,
7685                                                              error);
7686                         if (ret < 0)
7687                                 return ret;
7688                         action_flags |= MLX5_FLOW_ACTION_CT;
7689                         break;
7690                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
7691                         /* tunnel offload action was processed before
7692                          * list it here as a supported type
7693                          */
7694                         break;
7695                 default:
7696                         return rte_flow_error_set(error, ENOTSUP,
7697                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7698                                                   actions,
7699                                                   "action not supported");
7700                 }
7701         }
7702         /*
7703          * Validate actions in flow rules
7704          * - Explicit decap action is prohibited by the tunnel offload API.
7705          * - Drop action in tunnel steer rule is prohibited by the API.
7706          * - Application cannot use MARK action because it's value can mask
7707          *   tunnel default miss nitification.
7708          * - JUMP in tunnel match rule has no support in current PMD
7709          *   implementation.
7710          * - TAG & META are reserved for future uses.
7711          */
7712         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
7713                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
7714                                             MLX5_FLOW_ACTION_MARK     |
7715                                             MLX5_FLOW_ACTION_SET_TAG  |
7716                                             MLX5_FLOW_ACTION_SET_META |
7717                                             MLX5_FLOW_ACTION_DROP;
7718
7719                 if (action_flags & bad_actions_mask)
7720                         return rte_flow_error_set
7721                                         (error, EINVAL,
7722                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7723                                         "Invalid RTE action in tunnel "
7724                                         "set decap rule");
7725                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
7726                         return rte_flow_error_set
7727                                         (error, EINVAL,
7728                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7729                                         "tunnel set decap rule must terminate "
7730                                         "with JUMP");
7731                 if (!attr->ingress)
7732                         return rte_flow_error_set
7733                                         (error, EINVAL,
7734                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7735                                         "tunnel flows for ingress traffic only");
7736         }
7737         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
7738                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
7739                                             MLX5_FLOW_ACTION_MARK    |
7740                                             MLX5_FLOW_ACTION_SET_TAG |
7741                                             MLX5_FLOW_ACTION_SET_META;
7742
7743                 if (action_flags & bad_actions_mask)
7744                         return rte_flow_error_set
7745                                         (error, EINVAL,
7746                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7747                                         "Invalid RTE action in tunnel "
7748                                         "set match rule");
7749         }
7750         /*
7751          * Validate the drop action mutual exclusion with other actions.
7752          * Drop action is mutually-exclusive with any other action, except for
7753          * Count action.
7754          * Drop action compatibility with tunnel offload was already validated.
7755          */
7756         if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH |
7757                             MLX5_FLOW_ACTION_TUNNEL_MATCH));
7758         else if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
7759             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
7760                 return rte_flow_error_set(error, EINVAL,
7761                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7762                                           "Drop action is mutually-exclusive "
7763                                           "with any other action, except for "
7764                                           "Count action");
7765         /* Eswitch has few restrictions on using items and actions */
7766         if (attr->transfer) {
7767                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7768                     action_flags & MLX5_FLOW_ACTION_FLAG)
7769                         return rte_flow_error_set(error, ENOTSUP,
7770                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7771                                                   NULL,
7772                                                   "unsupported action FLAG");
7773                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7774                     action_flags & MLX5_FLOW_ACTION_MARK)
7775                         return rte_flow_error_set(error, ENOTSUP,
7776                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7777                                                   NULL,
7778                                                   "unsupported action MARK");
7779                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
7780                         return rte_flow_error_set(error, ENOTSUP,
7781                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7782                                                   NULL,
7783                                                   "unsupported action QUEUE");
7784                 if (action_flags & MLX5_FLOW_ACTION_RSS)
7785                         return rte_flow_error_set(error, ENOTSUP,
7786                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7787                                                   NULL,
7788                                                   "unsupported action RSS");
7789                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
7790                         return rte_flow_error_set(error, EINVAL,
7791                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7792                                                   actions,
7793                                                   "no fate action is found");
7794         } else {
7795                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
7796                         return rte_flow_error_set(error, EINVAL,
7797                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7798                                                   actions,
7799                                                   "no fate action is found");
7800         }
7801         /*
7802          * Continue validation for Xcap and VLAN actions.
7803          * If hairpin is working in explicit TX rule mode, there is no actions
7804          * splitting and the validation of hairpin ingress flow should be the
7805          * same as other standard flows.
7806          */
7807         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
7808                              MLX5_FLOW_VLAN_ACTIONS)) &&
7809             (queue_index == 0xFFFF ||
7810              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN ||
7811              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
7812              conf->tx_explicit != 0))) {
7813                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
7814                     MLX5_FLOW_XCAP_ACTIONS)
7815                         return rte_flow_error_set(error, ENOTSUP,
7816                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7817                                                   NULL, "encap and decap "
7818                                                   "combination aren't supported");
7819                 if (!attr->transfer && attr->ingress) {
7820                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7821                                 return rte_flow_error_set
7822                                                 (error, ENOTSUP,
7823                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7824                                                  NULL, "encap is not supported"
7825                                                  " for ingress traffic");
7826                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7827                                 return rte_flow_error_set
7828                                                 (error, ENOTSUP,
7829                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7830                                                  NULL, "push VLAN action not "
7831                                                  "supported for ingress");
7832                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
7833                                         MLX5_FLOW_VLAN_ACTIONS)
7834                                 return rte_flow_error_set
7835                                                 (error, ENOTSUP,
7836                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7837                                                  NULL, "no support for "
7838                                                  "multiple VLAN actions");
7839                 }
7840         }
7841         if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) {
7842                 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS &
7843                         ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) &&
7844                         attr->ingress)
7845                         return rte_flow_error_set
7846                                 (error, ENOTSUP,
7847                                 RTE_FLOW_ERROR_TYPE_ACTION,
7848                                 NULL, "fate action not supported for "
7849                                 "meter with policy");
7850                 if (attr->egress) {
7851                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
7852                                 return rte_flow_error_set
7853                                         (error, ENOTSUP,
7854                                         RTE_FLOW_ERROR_TYPE_ACTION,
7855                                         NULL, "modify header action in egress "
7856                                         "cannot be done before meter action");
7857                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7858                                 return rte_flow_error_set
7859                                         (error, ENOTSUP,
7860                                         RTE_FLOW_ERROR_TYPE_ACTION,
7861                                         NULL, "encap action in egress "
7862                                         "cannot be done before meter action");
7863                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7864                                 return rte_flow_error_set
7865                                         (error, ENOTSUP,
7866                                         RTE_FLOW_ERROR_TYPE_ACTION,
7867                                         NULL, "push vlan action in egress "
7868                                         "cannot be done before meter action");
7869                 }
7870         }
7871         /*
7872          * Hairpin flow will add one more TAG action in TX implicit mode.
7873          * In TX explicit mode, there will be no hairpin flow ID.
7874          */
7875         if (hairpin > 0)
7876                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7877         /* extra metadata enabled: one more TAG action will be add. */
7878         if (dev_conf->dv_flow_en &&
7879             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
7880             mlx5_flow_ext_mreg_supported(dev))
7881                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7882         if (rw_act_num >
7883                         flow_dv_modify_hdr_action_max(dev, is_root)) {
7884                 return rte_flow_error_set(error, ENOTSUP,
7885                                           RTE_FLOW_ERROR_TYPE_ACTION,
7886                                           NULL, "too many header modify"
7887                                           " actions to support");
7888         }
7889         /* Eswitch egress mirror and modify flow has limitation on CX5 */
7890         if (fdb_mirror_limit && modify_after_mirror)
7891                 return rte_flow_error_set(error, EINVAL,
7892                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7893                                 "sample before modify action is not supported");
7894         return 0;
7895 }
7896
7897 /**
7898  * Internal preparation function. Allocates the DV flow size,
7899  * this size is constant.
7900  *
7901  * @param[in] dev
7902  *   Pointer to the rte_eth_dev structure.
7903  * @param[in] attr
7904  *   Pointer to the flow attributes.
7905  * @param[in] items
7906  *   Pointer to the list of items.
7907  * @param[in] actions
7908  *   Pointer to the list of actions.
7909  * @param[out] error
7910  *   Pointer to the error structure.
7911  *
7912  * @return
7913  *   Pointer to mlx5_flow object on success,
7914  *   otherwise NULL and rte_errno is set.
7915  */
7916 static struct mlx5_flow *
7917 flow_dv_prepare(struct rte_eth_dev *dev,
7918                 const struct rte_flow_attr *attr __rte_unused,
7919                 const struct rte_flow_item items[] __rte_unused,
7920                 const struct rte_flow_action actions[] __rte_unused,
7921                 struct rte_flow_error *error)
7922 {
7923         uint32_t handle_idx = 0;
7924         struct mlx5_flow *dev_flow;
7925         struct mlx5_flow_handle *dev_handle;
7926         struct mlx5_priv *priv = dev->data->dev_private;
7927         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
7928
7929         MLX5_ASSERT(wks);
7930         wks->skip_matcher_reg = 0;
7931         wks->policy = NULL;
7932         wks->final_policy = NULL;
7933         /* In case of corrupting the memory. */
7934         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
7935                 rte_flow_error_set(error, ENOSPC,
7936                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7937                                    "not free temporary device flow");
7938                 return NULL;
7939         }
7940         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
7941                                    &handle_idx);
7942         if (!dev_handle) {
7943                 rte_flow_error_set(error, ENOMEM,
7944                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7945                                    "not enough memory to create flow handle");
7946                 return NULL;
7947         }
7948         MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows));
7949         dev_flow = &wks->flows[wks->flow_idx++];
7950         memset(dev_flow, 0, sizeof(*dev_flow));
7951         dev_flow->handle = dev_handle;
7952         dev_flow->handle_idx = handle_idx;
7953         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
7954         dev_flow->ingress = attr->ingress;
7955         dev_flow->dv.transfer = attr->transfer;
7956         return dev_flow;
7957 }
7958
7959 #ifdef RTE_LIBRTE_MLX5_DEBUG
7960 /**
7961  * Sanity check for match mask and value. Similar to check_valid_spec() in
7962  * kernel driver. If unmasked bit is present in value, it returns failure.
7963  *
7964  * @param match_mask
7965  *   pointer to match mask buffer.
7966  * @param match_value
7967  *   pointer to match value buffer.
7968  *
7969  * @return
7970  *   0 if valid, -EINVAL otherwise.
7971  */
7972 static int
7973 flow_dv_check_valid_spec(void *match_mask, void *match_value)
7974 {
7975         uint8_t *m = match_mask;
7976         uint8_t *v = match_value;
7977         unsigned int i;
7978
7979         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
7980                 if (v[i] & ~m[i]) {
7981                         DRV_LOG(ERR,
7982                                 "match_value differs from match_criteria"
7983                                 " %p[%u] != %p[%u]",
7984                                 match_value, i, match_mask, i);
7985                         return -EINVAL;
7986                 }
7987         }
7988         return 0;
7989 }
7990 #endif
7991
7992 /**
7993  * Add match of ip_version.
7994  *
7995  * @param[in] group
7996  *   Flow group.
7997  * @param[in] headers_v
7998  *   Values header pointer.
7999  * @param[in] headers_m
8000  *   Masks header pointer.
8001  * @param[in] ip_version
8002  *   The IP version to set.
8003  */
8004 static inline void
8005 flow_dv_set_match_ip_version(uint32_t group,
8006                              void *headers_v,
8007                              void *headers_m,
8008                              uint8_t ip_version)
8009 {
8010         if (group == 0)
8011                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
8012         else
8013                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
8014                          ip_version);
8015         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
8016         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
8017         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
8018 }
8019
8020 /**
8021  * Add Ethernet item to matcher and to the value.
8022  *
8023  * @param[in, out] matcher
8024  *   Flow matcher.
8025  * @param[in, out] key
8026  *   Flow matcher value.
8027  * @param[in] item
8028  *   Flow pattern to translate.
8029  * @param[in] inner
8030  *   Item is inner pattern.
8031  */
8032 static void
8033 flow_dv_translate_item_eth(void *matcher, void *key,
8034                            const struct rte_flow_item *item, int inner,
8035                            uint32_t group)
8036 {
8037         const struct rte_flow_item_eth *eth_m = item->mask;
8038         const struct rte_flow_item_eth *eth_v = item->spec;
8039         const struct rte_flow_item_eth nic_mask = {
8040                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8041                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8042                 .type = RTE_BE16(0xffff),
8043                 .has_vlan = 0,
8044         };
8045         void *hdrs_m;
8046         void *hdrs_v;
8047         char *l24_v;
8048         unsigned int i;
8049
8050         if (!eth_v)
8051                 return;
8052         if (!eth_m)
8053                 eth_m = &nic_mask;
8054         if (inner) {
8055                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8056                                          inner_headers);
8057                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8058         } else {
8059                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8060                                          outer_headers);
8061                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8062         }
8063         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
8064                &eth_m->dst, sizeof(eth_m->dst));
8065         /* The value must be in the range of the mask. */
8066         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
8067         for (i = 0; i < sizeof(eth_m->dst); ++i)
8068                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
8069         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
8070                &eth_m->src, sizeof(eth_m->src));
8071         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
8072         /* The value must be in the range of the mask. */
8073         for (i = 0; i < sizeof(eth_m->dst); ++i)
8074                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
8075         /*
8076          * HW supports match on one Ethertype, the Ethertype following the last
8077          * VLAN tag of the packet (see PRM).
8078          * Set match on ethertype only if ETH header is not followed by VLAN.
8079          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8080          * ethertype, and use ip_version field instead.
8081          * eCPRI over Ether layer will use type value 0xAEFE.
8082          */
8083         if (eth_m->type == 0xFFFF) {
8084                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
8085                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8086                 switch (eth_v->type) {
8087                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8088                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8089                         return;
8090                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
8091                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8092                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8093                         return;
8094                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8095                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8096                         return;
8097                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8098                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8099                         return;
8100                 default:
8101                         break;
8102                 }
8103         }
8104         if (eth_m->has_vlan) {
8105                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8106                 if (eth_v->has_vlan) {
8107                         /*
8108                          * Here, when also has_more_vlan field in VLAN item is
8109                          * not set, only single-tagged packets will be matched.
8110                          */
8111                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8112                         return;
8113                 }
8114         }
8115         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8116                  rte_be_to_cpu_16(eth_m->type));
8117         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
8118         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
8119 }
8120
8121 /**
8122  * Add VLAN item to matcher and to the value.
8123  *
8124  * @param[in, out] dev_flow
8125  *   Flow descriptor.
8126  * @param[in, out] matcher
8127  *   Flow matcher.
8128  * @param[in, out] key
8129  *   Flow matcher value.
8130  * @param[in] item
8131  *   Flow pattern to translate.
8132  * @param[in] inner
8133  *   Item is inner pattern.
8134  */
8135 static void
8136 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
8137                             void *matcher, void *key,
8138                             const struct rte_flow_item *item,
8139                             int inner, uint32_t group)
8140 {
8141         const struct rte_flow_item_vlan *vlan_m = item->mask;
8142         const struct rte_flow_item_vlan *vlan_v = item->spec;
8143         void *hdrs_m;
8144         void *hdrs_v;
8145         uint16_t tci_m;
8146         uint16_t tci_v;
8147
8148         if (inner) {
8149                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8150                                          inner_headers);
8151                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8152         } else {
8153                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8154                                          outer_headers);
8155                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8156                 /*
8157                  * This is workaround, masks are not supported,
8158                  * and pre-validated.
8159                  */
8160                 if (vlan_v)
8161                         dev_flow->handle->vf_vlan.tag =
8162                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
8163         }
8164         /*
8165          * When VLAN item exists in flow, mark packet as tagged,
8166          * even if TCI is not specified.
8167          */
8168         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
8169                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8170                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8171         }
8172         if (!vlan_v)
8173                 return;
8174         if (!vlan_m)
8175                 vlan_m = &rte_flow_item_vlan_mask;
8176         tci_m = rte_be_to_cpu_16(vlan_m->tci);
8177         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
8178         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
8179         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
8180         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
8181         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
8182         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
8183         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
8184         /*
8185          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8186          * ethertype, and use ip_version field instead.
8187          */
8188         if (vlan_m->inner_type == 0xFFFF) {
8189                 switch (vlan_v->inner_type) {
8190                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8191                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8192                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8193                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8194                         return;
8195                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8196                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8197                         return;
8198                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8199                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8200                         return;
8201                 default:
8202                         break;
8203                 }
8204         }
8205         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
8206                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8207                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8208                 /* Only one vlan_tag bit can be set. */
8209                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8210                 return;
8211         }
8212         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8213                  rte_be_to_cpu_16(vlan_m->inner_type));
8214         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
8215                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
8216 }
8217
8218 /**
8219  * Add IPV4 item to matcher and to the value.
8220  *
8221  * @param[in, out] matcher
8222  *   Flow matcher.
8223  * @param[in, out] key
8224  *   Flow matcher value.
8225  * @param[in] item
8226  *   Flow pattern to translate.
8227  * @param[in] inner
8228  *   Item is inner pattern.
8229  * @param[in] group
8230  *   The group to insert the rule.
8231  */
8232 static void
8233 flow_dv_translate_item_ipv4(void *matcher, void *key,
8234                             const struct rte_flow_item *item,
8235                             int inner, uint32_t group)
8236 {
8237         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
8238         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
8239         const struct rte_flow_item_ipv4 nic_mask = {
8240                 .hdr = {
8241                         .src_addr = RTE_BE32(0xffffffff),
8242                         .dst_addr = RTE_BE32(0xffffffff),
8243                         .type_of_service = 0xff,
8244                         .next_proto_id = 0xff,
8245                         .time_to_live = 0xff,
8246                 },
8247         };
8248         void *headers_m;
8249         void *headers_v;
8250         char *l24_m;
8251         char *l24_v;
8252         uint8_t tos;
8253
8254         if (inner) {
8255                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8256                                          inner_headers);
8257                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8258         } else {
8259                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8260                                          outer_headers);
8261                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8262         }
8263         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
8264         if (!ipv4_v)
8265                 return;
8266         if (!ipv4_m)
8267                 ipv4_m = &nic_mask;
8268         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8269                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8270         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8271                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8272         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
8273         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
8274         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8275                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8276         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8277                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8278         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
8279         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
8280         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
8281         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
8282                  ipv4_m->hdr.type_of_service);
8283         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
8284         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
8285                  ipv4_m->hdr.type_of_service >> 2);
8286         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
8287         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8288                  ipv4_m->hdr.next_proto_id);
8289         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8290                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
8291         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8292                  ipv4_m->hdr.time_to_live);
8293         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8294                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
8295         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8296                  !!(ipv4_m->hdr.fragment_offset));
8297         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8298                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
8299 }
8300
8301 /**
8302  * Add IPV6 item to matcher and to the value.
8303  *
8304  * @param[in, out] matcher
8305  *   Flow matcher.
8306  * @param[in, out] key
8307  *   Flow matcher value.
8308  * @param[in] item
8309  *   Flow pattern to translate.
8310  * @param[in] inner
8311  *   Item is inner pattern.
8312  * @param[in] group
8313  *   The group to insert the rule.
8314  */
8315 static void
8316 flow_dv_translate_item_ipv6(void *matcher, void *key,
8317                             const struct rte_flow_item *item,
8318                             int inner, uint32_t group)
8319 {
8320         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
8321         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
8322         const struct rte_flow_item_ipv6 nic_mask = {
8323                 .hdr = {
8324                         .src_addr =
8325                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8326                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8327                         .dst_addr =
8328                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8329                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8330                         .vtc_flow = RTE_BE32(0xffffffff),
8331                         .proto = 0xff,
8332                         .hop_limits = 0xff,
8333                 },
8334         };
8335         void *headers_m;
8336         void *headers_v;
8337         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8338         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8339         char *l24_m;
8340         char *l24_v;
8341         uint32_t vtc_m;
8342         uint32_t vtc_v;
8343         int i;
8344         int size;
8345
8346         if (inner) {
8347                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8348                                          inner_headers);
8349                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8350         } else {
8351                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8352                                          outer_headers);
8353                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8354         }
8355         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
8356         if (!ipv6_v)
8357                 return;
8358         if (!ipv6_m)
8359                 ipv6_m = &nic_mask;
8360         size = sizeof(ipv6_m->hdr.dst_addr);
8361         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8362                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8363         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8364                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8365         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
8366         for (i = 0; i < size; ++i)
8367                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
8368         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8369                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8370         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8371                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8372         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
8373         for (i = 0; i < size; ++i)
8374                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
8375         /* TOS. */
8376         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
8377         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
8378         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
8379         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
8380         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
8381         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
8382         /* Label. */
8383         if (inner) {
8384                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
8385                          vtc_m);
8386                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
8387                          vtc_v);
8388         } else {
8389                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
8390                          vtc_m);
8391                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
8392                          vtc_v);
8393         }
8394         /* Protocol. */
8395         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8396                  ipv6_m->hdr.proto);
8397         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8398                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
8399         /* Hop limit. */
8400         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8401                  ipv6_m->hdr.hop_limits);
8402         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8403                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
8404         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8405                  !!(ipv6_m->has_frag_ext));
8406         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8407                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
8408 }
8409
8410 /**
8411  * Add IPV6 fragment extension item to matcher and to the value.
8412  *
8413  * @param[in, out] matcher
8414  *   Flow matcher.
8415  * @param[in, out] key
8416  *   Flow matcher value.
8417  * @param[in] item
8418  *   Flow pattern to translate.
8419  * @param[in] inner
8420  *   Item is inner pattern.
8421  */
8422 static void
8423 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
8424                                      const struct rte_flow_item *item,
8425                                      int inner)
8426 {
8427         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
8428         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
8429         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
8430                 .hdr = {
8431                         .next_header = 0xff,
8432                         .frag_data = RTE_BE16(0xffff),
8433                 },
8434         };
8435         void *headers_m;
8436         void *headers_v;
8437
8438         if (inner) {
8439                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8440                                          inner_headers);
8441                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8442         } else {
8443                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8444                                          outer_headers);
8445                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8446         }
8447         /* IPv6 fragment extension item exists, so packet is IP fragment. */
8448         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
8449         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
8450         if (!ipv6_frag_ext_v)
8451                 return;
8452         if (!ipv6_frag_ext_m)
8453                 ipv6_frag_ext_m = &nic_mask;
8454         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8455                  ipv6_frag_ext_m->hdr.next_header);
8456         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8457                  ipv6_frag_ext_v->hdr.next_header &
8458                  ipv6_frag_ext_m->hdr.next_header);
8459 }
8460
8461 /**
8462  * Add TCP item to matcher and to the value.
8463  *
8464  * @param[in, out] matcher
8465  *   Flow matcher.
8466  * @param[in, out] key
8467  *   Flow matcher value.
8468  * @param[in] item
8469  *   Flow pattern to translate.
8470  * @param[in] inner
8471  *   Item is inner pattern.
8472  */
8473 static void
8474 flow_dv_translate_item_tcp(void *matcher, void *key,
8475                            const struct rte_flow_item *item,
8476                            int inner)
8477 {
8478         const struct rte_flow_item_tcp *tcp_m = item->mask;
8479         const struct rte_flow_item_tcp *tcp_v = item->spec;
8480         void *headers_m;
8481         void *headers_v;
8482
8483         if (inner) {
8484                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8485                                          inner_headers);
8486                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8487         } else {
8488                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8489                                          outer_headers);
8490                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8491         }
8492         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8493         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
8494         if (!tcp_v)
8495                 return;
8496         if (!tcp_m)
8497                 tcp_m = &rte_flow_item_tcp_mask;
8498         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
8499                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
8500         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
8501                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
8502         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
8503                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
8504         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
8505                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
8506         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
8507                  tcp_m->hdr.tcp_flags);
8508         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
8509                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
8510 }
8511
8512 /**
8513  * Add UDP item to matcher and to the value.
8514  *
8515  * @param[in, out] matcher
8516  *   Flow matcher.
8517  * @param[in, out] key
8518  *   Flow matcher value.
8519  * @param[in] item
8520  *   Flow pattern to translate.
8521  * @param[in] inner
8522  *   Item is inner pattern.
8523  */
8524 static void
8525 flow_dv_translate_item_udp(void *matcher, void *key,
8526                            const struct rte_flow_item *item,
8527                            int inner)
8528 {
8529         const struct rte_flow_item_udp *udp_m = item->mask;
8530         const struct rte_flow_item_udp *udp_v = item->spec;
8531         void *headers_m;
8532         void *headers_v;
8533
8534         if (inner) {
8535                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8536                                          inner_headers);
8537                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8538         } else {
8539                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8540                                          outer_headers);
8541                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8542         }
8543         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8544         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
8545         if (!udp_v)
8546                 return;
8547         if (!udp_m)
8548                 udp_m = &rte_flow_item_udp_mask;
8549         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
8550                  rte_be_to_cpu_16(udp_m->hdr.src_port));
8551         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
8552                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
8553         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
8554                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
8555         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
8556                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
8557 }
8558
8559 /**
8560  * Add GRE optional Key item to matcher and to the value.
8561  *
8562  * @param[in, out] matcher
8563  *   Flow matcher.
8564  * @param[in, out] key
8565  *   Flow matcher value.
8566  * @param[in] item
8567  *   Flow pattern to translate.
8568  * @param[in] inner
8569  *   Item is inner pattern.
8570  */
8571 static void
8572 flow_dv_translate_item_gre_key(void *matcher, void *key,
8573                                    const struct rte_flow_item *item)
8574 {
8575         const rte_be32_t *key_m = item->mask;
8576         const rte_be32_t *key_v = item->spec;
8577         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8578         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8579         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
8580
8581         /* GRE K bit must be on and should already be validated */
8582         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
8583         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
8584         if (!key_v)
8585                 return;
8586         if (!key_m)
8587                 key_m = &gre_key_default_mask;
8588         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
8589                  rte_be_to_cpu_32(*key_m) >> 8);
8590         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
8591                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
8592         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
8593                  rte_be_to_cpu_32(*key_m) & 0xFF);
8594         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
8595                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
8596 }
8597
8598 /**
8599  * Add GRE item to matcher and to the value.
8600  *
8601  * @param[in, out] matcher
8602  *   Flow matcher.
8603  * @param[in, out] key
8604  *   Flow matcher value.
8605  * @param[in] item
8606  *   Flow pattern to translate.
8607  * @param[in] inner
8608  *   Item is inner pattern.
8609  */
8610 static void
8611 flow_dv_translate_item_gre(void *matcher, void *key,
8612                            const struct rte_flow_item *item,
8613                            int inner)
8614 {
8615         const struct rte_flow_item_gre *gre_m = item->mask;
8616         const struct rte_flow_item_gre *gre_v = item->spec;
8617         void *headers_m;
8618         void *headers_v;
8619         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8620         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8621         struct {
8622                 union {
8623                         __extension__
8624                         struct {
8625                                 uint16_t version:3;
8626                                 uint16_t rsvd0:9;
8627                                 uint16_t s_present:1;
8628                                 uint16_t k_present:1;
8629                                 uint16_t rsvd_bit1:1;
8630                                 uint16_t c_present:1;
8631                         };
8632                         uint16_t value;
8633                 };
8634         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
8635
8636         if (inner) {
8637                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8638                                          inner_headers);
8639                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8640         } else {
8641                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8642                                          outer_headers);
8643                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8644         }
8645         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8646         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
8647         if (!gre_v)
8648                 return;
8649         if (!gre_m)
8650                 gre_m = &rte_flow_item_gre_mask;
8651         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
8652                  rte_be_to_cpu_16(gre_m->protocol));
8653         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
8654                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
8655         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
8656         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
8657         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
8658                  gre_crks_rsvd0_ver_m.c_present);
8659         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
8660                  gre_crks_rsvd0_ver_v.c_present &
8661                  gre_crks_rsvd0_ver_m.c_present);
8662         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
8663                  gre_crks_rsvd0_ver_m.k_present);
8664         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
8665                  gre_crks_rsvd0_ver_v.k_present &
8666                  gre_crks_rsvd0_ver_m.k_present);
8667         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
8668                  gre_crks_rsvd0_ver_m.s_present);
8669         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
8670                  gre_crks_rsvd0_ver_v.s_present &
8671                  gre_crks_rsvd0_ver_m.s_present);
8672 }
8673
8674 /**
8675  * Add NVGRE item to matcher and to the value.
8676  *
8677  * @param[in, out] matcher
8678  *   Flow matcher.
8679  * @param[in, out] key
8680  *   Flow matcher value.
8681  * @param[in] item
8682  *   Flow pattern to translate.
8683  * @param[in] inner
8684  *   Item is inner pattern.
8685  */
8686 static void
8687 flow_dv_translate_item_nvgre(void *matcher, void *key,
8688                              const struct rte_flow_item *item,
8689                              int inner)
8690 {
8691         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
8692         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
8693         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8694         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8695         const char *tni_flow_id_m;
8696         const char *tni_flow_id_v;
8697         char *gre_key_m;
8698         char *gre_key_v;
8699         int size;
8700         int i;
8701
8702         /* For NVGRE, GRE header fields must be set with defined values. */
8703         const struct rte_flow_item_gre gre_spec = {
8704                 .c_rsvd0_ver = RTE_BE16(0x2000),
8705                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
8706         };
8707         const struct rte_flow_item_gre gre_mask = {
8708                 .c_rsvd0_ver = RTE_BE16(0xB000),
8709                 .protocol = RTE_BE16(UINT16_MAX),
8710         };
8711         const struct rte_flow_item gre_item = {
8712                 .spec = &gre_spec,
8713                 .mask = &gre_mask,
8714                 .last = NULL,
8715         };
8716         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
8717         if (!nvgre_v)
8718                 return;
8719         if (!nvgre_m)
8720                 nvgre_m = &rte_flow_item_nvgre_mask;
8721         tni_flow_id_m = (const char *)nvgre_m->tni;
8722         tni_flow_id_v = (const char *)nvgre_v->tni;
8723         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
8724         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
8725         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
8726         memcpy(gre_key_m, tni_flow_id_m, size);
8727         for (i = 0; i < size; ++i)
8728                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
8729 }
8730
8731 /**
8732  * Add VXLAN item to matcher and to the value.
8733  *
8734  * @param[in] dev
8735  *   Pointer to the Ethernet device structure.
8736  * @param[in] attr
8737  *   Flow rule attributes.
8738  * @param[in, out] matcher
8739  *   Flow matcher.
8740  * @param[in, out] key
8741  *   Flow matcher value.
8742  * @param[in] item
8743  *   Flow pattern to translate.
8744  * @param[in] inner
8745  *   Item is inner pattern.
8746  */
8747 static void
8748 flow_dv_translate_item_vxlan(struct rte_eth_dev *dev,
8749                              const struct rte_flow_attr *attr,
8750                              void *matcher, void *key,
8751                              const struct rte_flow_item *item,
8752                              int inner)
8753 {
8754         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
8755         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
8756         void *headers_m;
8757         void *headers_v;
8758         void *misc5_m;
8759         void *misc5_v;
8760         uint32_t *tunnel_header_v;
8761         uint32_t *tunnel_header_m;
8762         uint16_t dport;
8763         struct mlx5_priv *priv = dev->data->dev_private;
8764         const struct rte_flow_item_vxlan nic_mask = {
8765                 .vni = "\xff\xff\xff",
8766                 .rsvd1 = 0xff,
8767         };
8768
8769         if (inner) {
8770                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8771                                          inner_headers);
8772                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8773         } else {
8774                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8775                                          outer_headers);
8776                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8777         }
8778         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8779                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8780         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8781                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8782                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8783         }
8784         if (!vxlan_v)
8785                 return;
8786         if (!vxlan_m) {
8787                 if ((!attr->group && !priv->sh->tunnel_header_0_1) ||
8788                     (attr->group && !priv->sh->misc5_cap))
8789                         vxlan_m = &rte_flow_item_vxlan_mask;
8790                 else
8791                         vxlan_m = &nic_mask;
8792         }
8793         if ((!attr->group && !attr->transfer && !priv->sh->tunnel_header_0_1) ||
8794             ((attr->group || attr->transfer) && !priv->sh->misc5_cap)) {
8795                 void *misc_m;
8796                 void *misc_v;
8797                 char *vni_m;
8798                 char *vni_v;
8799                 int size;
8800                 int i;
8801                 misc_m = MLX5_ADDR_OF(fte_match_param,
8802                                       matcher, misc_parameters);
8803                 misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8804                 size = sizeof(vxlan_m->vni);
8805                 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
8806                 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
8807                 memcpy(vni_m, vxlan_m->vni, size);
8808                 for (i = 0; i < size; ++i)
8809                         vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8810                 return;
8811         }
8812         misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
8813         misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
8814         tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8815                                                    misc5_v,
8816                                                    tunnel_header_1);
8817         tunnel_header_m = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8818                                                    misc5_m,
8819                                                    tunnel_header_1);
8820         *tunnel_header_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) |
8821                            (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 |
8822                            (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16;
8823         if (*tunnel_header_v)
8824                 *tunnel_header_m = vxlan_m->vni[0] |
8825                         vxlan_m->vni[1] << 8 |
8826                         vxlan_m->vni[2] << 16;
8827         else
8828                 *tunnel_header_m = 0x0;
8829         *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24;
8830         if (vxlan_v->rsvd1 & vxlan_m->rsvd1)
8831                 *tunnel_header_m |= vxlan_m->rsvd1 << 24;
8832 }
8833
8834 /**
8835  * Add VXLAN-GPE item to matcher and to the value.
8836  *
8837  * @param[in, out] matcher
8838  *   Flow matcher.
8839  * @param[in, out] key
8840  *   Flow matcher value.
8841  * @param[in] item
8842  *   Flow pattern to translate.
8843  * @param[in] inner
8844  *   Item is inner pattern.
8845  */
8846
8847 static void
8848 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
8849                                  const struct rte_flow_item *item, int inner)
8850 {
8851         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
8852         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
8853         void *headers_m;
8854         void *headers_v;
8855         void *misc_m =
8856                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
8857         void *misc_v =
8858                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
8859         char *vni_m;
8860         char *vni_v;
8861         uint16_t dport;
8862         int size;
8863         int i;
8864         uint8_t flags_m = 0xff;
8865         uint8_t flags_v = 0xc;
8866
8867         if (inner) {
8868                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8869                                          inner_headers);
8870                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8871         } else {
8872                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8873                                          outer_headers);
8874                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8875         }
8876         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8877                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8878         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8879                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8880                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8881         }
8882         if (!vxlan_v)
8883                 return;
8884         if (!vxlan_m)
8885                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
8886         size = sizeof(vxlan_m->vni);
8887         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
8888         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
8889         memcpy(vni_m, vxlan_m->vni, size);
8890         for (i = 0; i < size; ++i)
8891                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8892         if (vxlan_m->flags) {
8893                 flags_m = vxlan_m->flags;
8894                 flags_v = vxlan_v->flags;
8895         }
8896         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
8897         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
8898         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
8899                  vxlan_m->protocol);
8900         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
8901                  vxlan_v->protocol);
8902 }
8903
8904 /**
8905  * Add Geneve item to matcher and to the value.
8906  *
8907  * @param[in, out] matcher
8908  *   Flow matcher.
8909  * @param[in, out] key
8910  *   Flow matcher value.
8911  * @param[in] item
8912  *   Flow pattern to translate.
8913  * @param[in] inner
8914  *   Item is inner pattern.
8915  */
8916
8917 static void
8918 flow_dv_translate_item_geneve(void *matcher, void *key,
8919                               const struct rte_flow_item *item, int inner)
8920 {
8921         const struct rte_flow_item_geneve *geneve_m = item->mask;
8922         const struct rte_flow_item_geneve *geneve_v = item->spec;
8923         void *headers_m;
8924         void *headers_v;
8925         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8926         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8927         uint16_t dport;
8928         uint16_t gbhdr_m;
8929         uint16_t gbhdr_v;
8930         char *vni_m;
8931         char *vni_v;
8932         size_t size, i;
8933
8934         if (inner) {
8935                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8936                                          inner_headers);
8937                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8938         } else {
8939                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8940                                          outer_headers);
8941                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8942         }
8943         dport = MLX5_UDP_PORT_GENEVE;
8944         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8945                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8946                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8947         }
8948         if (!geneve_v)
8949                 return;
8950         if (!geneve_m)
8951                 geneve_m = &rte_flow_item_geneve_mask;
8952         size = sizeof(geneve_m->vni);
8953         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
8954         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
8955         memcpy(vni_m, geneve_m->vni, size);
8956         for (i = 0; i < size; ++i)
8957                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
8958         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
8959                  rte_be_to_cpu_16(geneve_m->protocol));
8960         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
8961                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
8962         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
8963         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
8964         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
8965                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
8966         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
8967                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
8968         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
8969                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
8970         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
8971                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
8972                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
8973 }
8974
8975 /**
8976  * Create Geneve TLV option resource.
8977  *
8978  * @param dev[in, out]
8979  *   Pointer to rte_eth_dev structure.
8980  * @param[in, out] tag_be24
8981  *   Tag value in big endian then R-shift 8.
8982  * @parm[in, out] dev_flow
8983  *   Pointer to the dev_flow.
8984  * @param[out] error
8985  *   pointer to error structure.
8986  *
8987  * @return
8988  *   0 on success otherwise -errno and errno is set.
8989  */
8990
8991 int
8992 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
8993                                              const struct rte_flow_item *item,
8994                                              struct rte_flow_error *error)
8995 {
8996         struct mlx5_priv *priv = dev->data->dev_private;
8997         struct mlx5_dev_ctx_shared *sh = priv->sh;
8998         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
8999                         sh->geneve_tlv_option_resource;
9000         struct mlx5_devx_obj *obj;
9001         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9002         int ret = 0;
9003
9004         if (!geneve_opt_v)
9005                 return -1;
9006         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
9007         if (geneve_opt_resource != NULL) {
9008                 if (geneve_opt_resource->option_class ==
9009                         geneve_opt_v->option_class &&
9010                         geneve_opt_resource->option_type ==
9011                         geneve_opt_v->option_type &&
9012                         geneve_opt_resource->length ==
9013                         geneve_opt_v->option_len) {
9014                         /* We already have GENVE TLV option obj allocated. */
9015                         __atomic_fetch_add(&geneve_opt_resource->refcnt, 1,
9016                                            __ATOMIC_RELAXED);
9017                 } else {
9018                         ret = rte_flow_error_set(error, ENOMEM,
9019                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9020                                 "Only one GENEVE TLV option supported");
9021                         goto exit;
9022                 }
9023         } else {
9024                 /* Create a GENEVE TLV object and resource. */
9025                 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->ctx,
9026                                 geneve_opt_v->option_class,
9027                                 geneve_opt_v->option_type,
9028                                 geneve_opt_v->option_len);
9029                 if (!obj) {
9030                         ret = rte_flow_error_set(error, ENODATA,
9031                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9032                                 "Failed to create GENEVE TLV Devx object");
9033                         goto exit;
9034                 }
9035                 sh->geneve_tlv_option_resource =
9036                                 mlx5_malloc(MLX5_MEM_ZERO,
9037                                                 sizeof(*geneve_opt_resource),
9038                                                 0, SOCKET_ID_ANY);
9039                 if (!sh->geneve_tlv_option_resource) {
9040                         claim_zero(mlx5_devx_cmd_destroy(obj));
9041                         ret = rte_flow_error_set(error, ENOMEM,
9042                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9043                                 "GENEVE TLV object memory allocation failed");
9044                         goto exit;
9045                 }
9046                 geneve_opt_resource = sh->geneve_tlv_option_resource;
9047                 geneve_opt_resource->obj = obj;
9048                 geneve_opt_resource->option_class = geneve_opt_v->option_class;
9049                 geneve_opt_resource->option_type = geneve_opt_v->option_type;
9050                 geneve_opt_resource->length = geneve_opt_v->option_len;
9051                 __atomic_store_n(&geneve_opt_resource->refcnt, 1,
9052                                 __ATOMIC_RELAXED);
9053         }
9054 exit:
9055         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
9056         return ret;
9057 }
9058
9059 /**
9060  * Add Geneve TLV option item to matcher.
9061  *
9062  * @param[in, out] dev
9063  *   Pointer to rte_eth_dev structure.
9064  * @param[in, out] matcher
9065  *   Flow matcher.
9066  * @param[in, out] key
9067  *   Flow matcher value.
9068  * @param[in] item
9069  *   Flow pattern to translate.
9070  * @param[out] error
9071  *   Pointer to error structure.
9072  */
9073 static int
9074 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher,
9075                                   void *key, const struct rte_flow_item *item,
9076                                   struct rte_flow_error *error)
9077 {
9078         const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask;
9079         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9080         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9081         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9082         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9083                         misc_parameters_3);
9084         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9085         rte_be32_t opt_data_key = 0, opt_data_mask = 0;
9086         int ret = 0;
9087
9088         if (!geneve_opt_v)
9089                 return -1;
9090         if (!geneve_opt_m)
9091                 geneve_opt_m = &rte_flow_item_geneve_opt_mask;
9092         ret = flow_dev_geneve_tlv_option_resource_register(dev, item,
9093                                                            error);
9094         if (ret) {
9095                 DRV_LOG(ERR, "Failed to create geneve_tlv_obj");
9096                 return ret;
9097         }
9098         /*
9099          * Set the option length in GENEVE header if not requested.
9100          * The GENEVE TLV option length is expressed by the option length field
9101          * in the GENEVE header.
9102          * If the option length was not requested but the GENEVE TLV option item
9103          * is present we set the option length field implicitly.
9104          */
9105         if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) {
9106                 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9107                          MLX5_GENEVE_OPTLEN_MASK);
9108                 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9109                          geneve_opt_v->option_len + 1);
9110         }
9111         /* Set the data. */
9112         if (geneve_opt_v->data) {
9113                 memcpy(&opt_data_key, geneve_opt_v->data,
9114                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9115                                 sizeof(opt_data_key)));
9116                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9117                                 sizeof(opt_data_key));
9118                 memcpy(&opt_data_mask, geneve_opt_m->data,
9119                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9120                                 sizeof(opt_data_mask)));
9121                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9122                                 sizeof(opt_data_mask));
9123                 MLX5_SET(fte_match_set_misc3, misc3_m,
9124                                 geneve_tlv_option_0_data,
9125                                 rte_be_to_cpu_32(opt_data_mask));
9126                 MLX5_SET(fte_match_set_misc3, misc3_v,
9127                                 geneve_tlv_option_0_data,
9128                         rte_be_to_cpu_32(opt_data_key & opt_data_mask));
9129         }
9130         return ret;
9131 }
9132
9133 /**
9134  * Add MPLS item to matcher and to the value.
9135  *
9136  * @param[in, out] matcher
9137  *   Flow matcher.
9138  * @param[in, out] key
9139  *   Flow matcher value.
9140  * @param[in] item
9141  *   Flow pattern to translate.
9142  * @param[in] prev_layer
9143  *   The protocol layer indicated in previous item.
9144  * @param[in] inner
9145  *   Item is inner pattern.
9146  */
9147 static void
9148 flow_dv_translate_item_mpls(void *matcher, void *key,
9149                             const struct rte_flow_item *item,
9150                             uint64_t prev_layer,
9151                             int inner)
9152 {
9153         const uint32_t *in_mpls_m = item->mask;
9154         const uint32_t *in_mpls_v = item->spec;
9155         uint32_t *out_mpls_m = 0;
9156         uint32_t *out_mpls_v = 0;
9157         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9158         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9159         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
9160                                      misc_parameters_2);
9161         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9162         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9163         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9164
9165         switch (prev_layer) {
9166         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9167                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
9168                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9169                          MLX5_UDP_PORT_MPLS);
9170                 break;
9171         case MLX5_FLOW_LAYER_GRE:
9172                 /* Fall-through. */
9173         case MLX5_FLOW_LAYER_GRE_KEY:
9174                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
9175                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
9176                          RTE_ETHER_TYPE_MPLS);
9177                 break;
9178         default:
9179                 break;
9180         }
9181         if (!in_mpls_v)
9182                 return;
9183         if (!in_mpls_m)
9184                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
9185         switch (prev_layer) {
9186         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9187                 out_mpls_m =
9188                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9189                                                  outer_first_mpls_over_udp);
9190                 out_mpls_v =
9191                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9192                                                  outer_first_mpls_over_udp);
9193                 break;
9194         case MLX5_FLOW_LAYER_GRE:
9195                 out_mpls_m =
9196                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9197                                                  outer_first_mpls_over_gre);
9198                 out_mpls_v =
9199                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9200                                                  outer_first_mpls_over_gre);
9201                 break;
9202         default:
9203                 /* Inner MPLS not over GRE is not supported. */
9204                 if (!inner) {
9205                         out_mpls_m =
9206                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9207                                                          misc2_m,
9208                                                          outer_first_mpls);
9209                         out_mpls_v =
9210                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9211                                                          misc2_v,
9212                                                          outer_first_mpls);
9213                 }
9214                 break;
9215         }
9216         if (out_mpls_m && out_mpls_v) {
9217                 *out_mpls_m = *in_mpls_m;
9218                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
9219         }
9220 }
9221
9222 /**
9223  * Add metadata register item to matcher
9224  *
9225  * @param[in, out] matcher
9226  *   Flow matcher.
9227  * @param[in, out] key
9228  *   Flow matcher value.
9229  * @param[in] reg_type
9230  *   Type of device metadata register
9231  * @param[in] value
9232  *   Register value
9233  * @param[in] mask
9234  *   Register mask
9235  */
9236 static void
9237 flow_dv_match_meta_reg(void *matcher, void *key,
9238                        enum modify_reg reg_type,
9239                        uint32_t data, uint32_t mask)
9240 {
9241         void *misc2_m =
9242                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
9243         void *misc2_v =
9244                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9245         uint32_t temp;
9246
9247         data &= mask;
9248         switch (reg_type) {
9249         case REG_A:
9250                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
9251                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
9252                 break;
9253         case REG_B:
9254                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
9255                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
9256                 break;
9257         case REG_C_0:
9258                 /*
9259                  * The metadata register C0 field might be divided into
9260                  * source vport index and META item value, we should set
9261                  * this field according to specified mask, not as whole one.
9262                  */
9263                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
9264                 temp |= mask;
9265                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
9266                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
9267                 temp &= ~mask;
9268                 temp |= data;
9269                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
9270                 break;
9271         case REG_C_1:
9272                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
9273                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
9274                 break;
9275         case REG_C_2:
9276                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
9277                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
9278                 break;
9279         case REG_C_3:
9280                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
9281                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
9282                 break;
9283         case REG_C_4:
9284                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
9285                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
9286                 break;
9287         case REG_C_5:
9288                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
9289                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
9290                 break;
9291         case REG_C_6:
9292                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
9293                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
9294                 break;
9295         case REG_C_7:
9296                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
9297                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
9298                 break;
9299         default:
9300                 MLX5_ASSERT(false);
9301                 break;
9302         }
9303 }
9304
9305 /**
9306  * Add MARK item to matcher
9307  *
9308  * @param[in] dev
9309  *   The device to configure through.
9310  * @param[in, out] matcher
9311  *   Flow matcher.
9312  * @param[in, out] key
9313  *   Flow matcher value.
9314  * @param[in] item
9315  *   Flow pattern to translate.
9316  */
9317 static void
9318 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
9319                             void *matcher, void *key,
9320                             const struct rte_flow_item *item)
9321 {
9322         struct mlx5_priv *priv = dev->data->dev_private;
9323         const struct rte_flow_item_mark *mark;
9324         uint32_t value;
9325         uint32_t mask;
9326
9327         mark = item->mask ? (const void *)item->mask :
9328                             &rte_flow_item_mark_mask;
9329         mask = mark->id & priv->sh->dv_mark_mask;
9330         mark = (const void *)item->spec;
9331         MLX5_ASSERT(mark);
9332         value = mark->id & priv->sh->dv_mark_mask & mask;
9333         if (mask) {
9334                 enum modify_reg reg;
9335
9336                 /* Get the metadata register index for the mark. */
9337                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
9338                 MLX5_ASSERT(reg > 0);
9339                 if (reg == REG_C_0) {
9340                         struct mlx5_priv *priv = dev->data->dev_private;
9341                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9342                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9343
9344                         mask &= msk_c0;
9345                         mask <<= shl_c0;
9346                         value <<= shl_c0;
9347                 }
9348                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9349         }
9350 }
9351
9352 /**
9353  * Add META item to matcher
9354  *
9355  * @param[in] dev
9356  *   The devich to configure through.
9357  * @param[in, out] matcher
9358  *   Flow matcher.
9359  * @param[in, out] key
9360  *   Flow matcher value.
9361  * @param[in] attr
9362  *   Attributes of flow that includes this item.
9363  * @param[in] item
9364  *   Flow pattern to translate.
9365  */
9366 static void
9367 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
9368                             void *matcher, void *key,
9369                             const struct rte_flow_attr *attr,
9370                             const struct rte_flow_item *item)
9371 {
9372         const struct rte_flow_item_meta *meta_m;
9373         const struct rte_flow_item_meta *meta_v;
9374
9375         meta_m = (const void *)item->mask;
9376         if (!meta_m)
9377                 meta_m = &rte_flow_item_meta_mask;
9378         meta_v = (const void *)item->spec;
9379         if (meta_v) {
9380                 int reg;
9381                 uint32_t value = meta_v->data;
9382                 uint32_t mask = meta_m->data;
9383
9384                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
9385                 if (reg < 0)
9386                         return;
9387                 MLX5_ASSERT(reg != REG_NON);
9388                 if (reg == REG_C_0) {
9389                         struct mlx5_priv *priv = dev->data->dev_private;
9390                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9391                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9392
9393                         mask &= msk_c0;
9394                         mask <<= shl_c0;
9395                         value <<= shl_c0;
9396                 }
9397                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9398         }
9399 }
9400
9401 /**
9402  * Add vport metadata Reg C0 item to matcher
9403  *
9404  * @param[in, out] matcher
9405  *   Flow matcher.
9406  * @param[in, out] key
9407  *   Flow matcher value.
9408  * @param[in] reg
9409  *   Flow pattern to translate.
9410  */
9411 static void
9412 flow_dv_translate_item_meta_vport(void *matcher, void *key,
9413                                   uint32_t value, uint32_t mask)
9414 {
9415         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
9416 }
9417
9418 /**
9419  * Add tag item to matcher
9420  *
9421  * @param[in] dev
9422  *   The devich to configure through.
9423  * @param[in, out] matcher
9424  *   Flow matcher.
9425  * @param[in, out] key
9426  *   Flow matcher value.
9427  * @param[in] item
9428  *   Flow pattern to translate.
9429  */
9430 static void
9431 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
9432                                 void *matcher, void *key,
9433                                 const struct rte_flow_item *item)
9434 {
9435         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
9436         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
9437         uint32_t mask, value;
9438
9439         MLX5_ASSERT(tag_v);
9440         value = tag_v->data;
9441         mask = tag_m ? tag_m->data : UINT32_MAX;
9442         if (tag_v->id == REG_C_0) {
9443                 struct mlx5_priv *priv = dev->data->dev_private;
9444                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9445                 uint32_t shl_c0 = rte_bsf32(msk_c0);
9446
9447                 mask &= msk_c0;
9448                 mask <<= shl_c0;
9449                 value <<= shl_c0;
9450         }
9451         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
9452 }
9453
9454 /**
9455  * Add TAG item to matcher
9456  *
9457  * @param[in] dev
9458  *   The devich to configure through.
9459  * @param[in, out] matcher
9460  *   Flow matcher.
9461  * @param[in, out] key
9462  *   Flow matcher value.
9463  * @param[in] item
9464  *   Flow pattern to translate.
9465  */
9466 static void
9467 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
9468                            void *matcher, void *key,
9469                            const struct rte_flow_item *item)
9470 {
9471         const struct rte_flow_item_tag *tag_v = item->spec;
9472         const struct rte_flow_item_tag *tag_m = item->mask;
9473         enum modify_reg reg;
9474
9475         MLX5_ASSERT(tag_v);
9476         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
9477         /* Get the metadata register index for the tag. */
9478         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
9479         MLX5_ASSERT(reg > 0);
9480         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
9481 }
9482
9483 /**
9484  * Add source vport match to the specified matcher.
9485  *
9486  * @param[in, out] matcher
9487  *   Flow matcher.
9488  * @param[in, out] key
9489  *   Flow matcher value.
9490  * @param[in] port
9491  *   Source vport value to match
9492  * @param[in] mask
9493  *   Mask
9494  */
9495 static void
9496 flow_dv_translate_item_source_vport(void *matcher, void *key,
9497                                     int16_t port, uint16_t mask)
9498 {
9499         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9500         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9501
9502         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
9503         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
9504 }
9505
9506 /**
9507  * Translate port-id item to eswitch match on  port-id.
9508  *
9509  * @param[in] dev
9510  *   The devich to configure through.
9511  * @param[in, out] matcher
9512  *   Flow matcher.
9513  * @param[in, out] key
9514  *   Flow matcher value.
9515  * @param[in] item
9516  *   Flow pattern to translate.
9517  * @param[in]
9518  *   Flow attributes.
9519  *
9520  * @return
9521  *   0 on success, a negative errno value otherwise.
9522  */
9523 static int
9524 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
9525                                void *key, const struct rte_flow_item *item,
9526                                const struct rte_flow_attr *attr)
9527 {
9528         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
9529         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
9530         struct mlx5_priv *priv;
9531         uint16_t mask, id;
9532
9533         mask = pid_m ? pid_m->id : 0xffff;
9534         id = pid_v ? pid_v->id : dev->data->port_id;
9535         priv = mlx5_port_to_eswitch_info(id, item == NULL);
9536         if (!priv)
9537                 return -rte_errno;
9538         /*
9539          * Translate to vport field or to metadata, depending on mode.
9540          * Kernel can use either misc.source_port or half of C0 metadata
9541          * register.
9542          */
9543         if (priv->vport_meta_mask) {
9544                 /*
9545                  * Provide the hint for SW steering library
9546                  * to insert the flow into ingress domain and
9547                  * save the extra vport match.
9548                  */
9549                 if (mask == 0xffff && priv->vport_id == 0xffff &&
9550                     priv->pf_bond < 0 && attr->transfer)
9551                         flow_dv_translate_item_source_vport
9552                                 (matcher, key, priv->vport_id, mask);
9553                 /*
9554                  * We should always set the vport metadata register,
9555                  * otherwise the SW steering library can drop
9556                  * the rule if wire vport metadata value is not zero,
9557                  * it depends on kernel configuration.
9558                  */
9559                 flow_dv_translate_item_meta_vport(matcher, key,
9560                                                   priv->vport_meta_tag,
9561                                                   priv->vport_meta_mask);
9562         } else {
9563                 flow_dv_translate_item_source_vport(matcher, key,
9564                                                     priv->vport_id, mask);
9565         }
9566         return 0;
9567 }
9568
9569 /**
9570  * Add ICMP6 item to matcher and to the value.
9571  *
9572  * @param[in, out] matcher
9573  *   Flow matcher.
9574  * @param[in, out] key
9575  *   Flow matcher value.
9576  * @param[in] item
9577  *   Flow pattern to translate.
9578  * @param[in] inner
9579  *   Item is inner pattern.
9580  */
9581 static void
9582 flow_dv_translate_item_icmp6(void *matcher, void *key,
9583                               const struct rte_flow_item *item,
9584                               int inner)
9585 {
9586         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
9587         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
9588         void *headers_m;
9589         void *headers_v;
9590         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9591                                      misc_parameters_3);
9592         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9593         if (inner) {
9594                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9595                                          inner_headers);
9596                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9597         } else {
9598                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9599                                          outer_headers);
9600                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9601         }
9602         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9603         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
9604         if (!icmp6_v)
9605                 return;
9606         if (!icmp6_m)
9607                 icmp6_m = &rte_flow_item_icmp6_mask;
9608         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
9609         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
9610                  icmp6_v->type & icmp6_m->type);
9611         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
9612         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
9613                  icmp6_v->code & icmp6_m->code);
9614 }
9615
9616 /**
9617  * Add ICMP item to matcher and to the value.
9618  *
9619  * @param[in, out] matcher
9620  *   Flow matcher.
9621  * @param[in, out] key
9622  *   Flow matcher value.
9623  * @param[in] item
9624  *   Flow pattern to translate.
9625  * @param[in] inner
9626  *   Item is inner pattern.
9627  */
9628 static void
9629 flow_dv_translate_item_icmp(void *matcher, void *key,
9630                             const struct rte_flow_item *item,
9631                             int inner)
9632 {
9633         const struct rte_flow_item_icmp *icmp_m = item->mask;
9634         const struct rte_flow_item_icmp *icmp_v = item->spec;
9635         uint32_t icmp_header_data_m = 0;
9636         uint32_t icmp_header_data_v = 0;
9637         void *headers_m;
9638         void *headers_v;
9639         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9640                                      misc_parameters_3);
9641         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9642         if (inner) {
9643                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9644                                          inner_headers);
9645                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9646         } else {
9647                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9648                                          outer_headers);
9649                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9650         }
9651         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9652         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
9653         if (!icmp_v)
9654                 return;
9655         if (!icmp_m)
9656                 icmp_m = &rte_flow_item_icmp_mask;
9657         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
9658                  icmp_m->hdr.icmp_type);
9659         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
9660                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
9661         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
9662                  icmp_m->hdr.icmp_code);
9663         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
9664                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
9665         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
9666         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
9667         if (icmp_header_data_m) {
9668                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
9669                 icmp_header_data_v |=
9670                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
9671                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
9672                          icmp_header_data_m);
9673                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
9674                          icmp_header_data_v & icmp_header_data_m);
9675         }
9676 }
9677
9678 /**
9679  * Add GTP item to matcher and to the value.
9680  *
9681  * @param[in, out] matcher
9682  *   Flow matcher.
9683  * @param[in, out] key
9684  *   Flow matcher value.
9685  * @param[in] item
9686  *   Flow pattern to translate.
9687  * @param[in] inner
9688  *   Item is inner pattern.
9689  */
9690 static void
9691 flow_dv_translate_item_gtp(void *matcher, void *key,
9692                            const struct rte_flow_item *item, int inner)
9693 {
9694         const struct rte_flow_item_gtp *gtp_m = item->mask;
9695         const struct rte_flow_item_gtp *gtp_v = item->spec;
9696         void *headers_m;
9697         void *headers_v;
9698         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9699                                      misc_parameters_3);
9700         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9701         uint16_t dport = RTE_GTPU_UDP_PORT;
9702
9703         if (inner) {
9704                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9705                                          inner_headers);
9706                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9707         } else {
9708                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9709                                          outer_headers);
9710                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9711         }
9712         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9713                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9714                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9715         }
9716         if (!gtp_v)
9717                 return;
9718         if (!gtp_m)
9719                 gtp_m = &rte_flow_item_gtp_mask;
9720         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
9721                  gtp_m->v_pt_rsv_flags);
9722         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
9723                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
9724         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
9725         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
9726                  gtp_v->msg_type & gtp_m->msg_type);
9727         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
9728                  rte_be_to_cpu_32(gtp_m->teid));
9729         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
9730                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
9731 }
9732
9733 /**
9734  * Add GTP PSC item to matcher.
9735  *
9736  * @param[in, out] matcher
9737  *   Flow matcher.
9738  * @param[in, out] key
9739  *   Flow matcher value.
9740  * @param[in] item
9741  *   Flow pattern to translate.
9742  */
9743 static int
9744 flow_dv_translate_item_gtp_psc(void *matcher, void *key,
9745                                const struct rte_flow_item *item)
9746 {
9747         const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask;
9748         const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec;
9749         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9750                         misc_parameters_3);
9751         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9752         union {
9753                 uint32_t w32;
9754                 struct {
9755                         uint16_t seq_num;
9756                         uint8_t npdu_num;
9757                         uint8_t next_ext_header_type;
9758                 };
9759         } dw_2;
9760         uint8_t gtp_flags;
9761
9762         /* Always set E-flag match on one, regardless of GTP item settings. */
9763         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags);
9764         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9765         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags);
9766         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags);
9767         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9768         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags);
9769         /*Set next extension header type. */
9770         dw_2.seq_num = 0;
9771         dw_2.npdu_num = 0;
9772         dw_2.next_ext_header_type = 0xff;
9773         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2,
9774                  rte_cpu_to_be_32(dw_2.w32));
9775         dw_2.seq_num = 0;
9776         dw_2.npdu_num = 0;
9777         dw_2.next_ext_header_type = 0x85;
9778         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2,
9779                  rte_cpu_to_be_32(dw_2.w32));
9780         if (gtp_psc_v) {
9781                 union {
9782                         uint32_t w32;
9783                         struct {
9784                                 uint8_t len;
9785                                 uint8_t type_flags;
9786                                 uint8_t qfi;
9787                                 uint8_t reserved;
9788                         };
9789                 } dw_0;
9790
9791                 /*Set extension header PDU type and Qos. */
9792                 if (!gtp_psc_m)
9793                         gtp_psc_m = &rte_flow_item_gtp_psc_mask;
9794                 dw_0.w32 = 0;
9795                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->pdu_type);
9796                 dw_0.qfi = gtp_psc_m->qfi;
9797                 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0,
9798                          rte_cpu_to_be_32(dw_0.w32));
9799                 dw_0.w32 = 0;
9800                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->pdu_type &
9801                                                         gtp_psc_m->pdu_type);
9802                 dw_0.qfi = gtp_psc_v->qfi & gtp_psc_m->qfi;
9803                 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0,
9804                          rte_cpu_to_be_32(dw_0.w32));
9805         }
9806         return 0;
9807 }
9808
9809 /**
9810  * Add eCPRI item to matcher and to the value.
9811  *
9812  * @param[in] dev
9813  *   The devich to configure through.
9814  * @param[in, out] matcher
9815  *   Flow matcher.
9816  * @param[in, out] key
9817  *   Flow matcher value.
9818  * @param[in] item
9819  *   Flow pattern to translate.
9820  * @param[in] samples
9821  *   Sample IDs to be used in the matching.
9822  */
9823 static void
9824 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
9825                              void *key, const struct rte_flow_item *item)
9826 {
9827         struct mlx5_priv *priv = dev->data->dev_private;
9828         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
9829         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
9830         struct rte_ecpri_common_hdr common;
9831         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
9832                                      misc_parameters_4);
9833         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
9834         uint32_t *samples;
9835         void *dw_m;
9836         void *dw_v;
9837
9838         if (!ecpri_v)
9839                 return;
9840         if (!ecpri_m)
9841                 ecpri_m = &rte_flow_item_ecpri_mask;
9842         /*
9843          * Maximal four DW samples are supported in a single matching now.
9844          * Two are used now for a eCPRI matching:
9845          * 1. Type: one byte, mask should be 0x00ff0000 in network order
9846          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
9847          *    if any.
9848          */
9849         if (!ecpri_m->hdr.common.u32)
9850                 return;
9851         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
9852         /* Need to take the whole DW as the mask to fill the entry. */
9853         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9854                             prog_sample_field_value_0);
9855         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9856                             prog_sample_field_value_0);
9857         /* Already big endian (network order) in the header. */
9858         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
9859         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32;
9860         /* Sample#0, used for matching type, offset 0. */
9861         MLX5_SET(fte_match_set_misc4, misc4_m,
9862                  prog_sample_field_id_0, samples[0]);
9863         /* It makes no sense to set the sample ID in the mask field. */
9864         MLX5_SET(fte_match_set_misc4, misc4_v,
9865                  prog_sample_field_id_0, samples[0]);
9866         /*
9867          * Checking if message body part needs to be matched.
9868          * Some wildcard rules only matching type field should be supported.
9869          */
9870         if (ecpri_m->hdr.dummy[0]) {
9871                 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32);
9872                 switch (common.type) {
9873                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
9874                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
9875                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
9876                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9877                                             prog_sample_field_value_1);
9878                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9879                                             prog_sample_field_value_1);
9880                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
9881                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] &
9882                                             ecpri_m->hdr.dummy[0];
9883                         /* Sample#1, to match message body, offset 4. */
9884                         MLX5_SET(fte_match_set_misc4, misc4_m,
9885                                  prog_sample_field_id_1, samples[1]);
9886                         MLX5_SET(fte_match_set_misc4, misc4_v,
9887                                  prog_sample_field_id_1, samples[1]);
9888                         break;
9889                 default:
9890                         /* Others, do not match any sample ID. */
9891                         break;
9892                 }
9893         }
9894 }
9895
9896 /*
9897  * Add connection tracking status item to matcher
9898  *
9899  * @param[in] dev
9900  *   The devich to configure through.
9901  * @param[in, out] matcher
9902  *   Flow matcher.
9903  * @param[in, out] key
9904  *   Flow matcher value.
9905  * @param[in] item
9906  *   Flow pattern to translate.
9907  */
9908 static void
9909 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev,
9910                               void *matcher, void *key,
9911                               const struct rte_flow_item *item)
9912 {
9913         uint32_t reg_value = 0;
9914         int reg_id;
9915         /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */
9916         uint32_t reg_mask = 0;
9917         const struct rte_flow_item_conntrack *spec = item->spec;
9918         const struct rte_flow_item_conntrack *mask = item->mask;
9919         uint32_t flags;
9920         struct rte_flow_error error;
9921
9922         if (!mask)
9923                 mask = &rte_flow_item_conntrack_mask;
9924         if (!spec || !mask->flags)
9925                 return;
9926         flags = spec->flags & mask->flags;
9927         /* The conflict should be checked in the validation. */
9928         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID)
9929                 reg_value |= MLX5_CT_SYNDROME_VALID;
9930         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
9931                 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE;
9932         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID)
9933                 reg_value |= MLX5_CT_SYNDROME_INVALID;
9934         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)
9935                 reg_value |= MLX5_CT_SYNDROME_TRAP;
9936         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
9937                 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET;
9938         if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID |
9939                            RTE_FLOW_CONNTRACK_PKT_STATE_INVALID |
9940                            RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))
9941                 reg_mask |= 0xc0;
9942         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
9943                 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE;
9944         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
9945                 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET;
9946         /* The REG_C_x value could be saved during startup. */
9947         reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error);
9948         if (reg_id == REG_NON)
9949                 return;
9950         flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id,
9951                                reg_value, reg_mask);
9952 }
9953
9954 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
9955
9956 #define HEADER_IS_ZERO(match_criteria, headers)                              \
9957         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
9958                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
9959
9960 /**
9961  * Calculate flow matcher enable bitmap.
9962  *
9963  * @param match_criteria
9964  *   Pointer to flow matcher criteria.
9965  *
9966  * @return
9967  *   Bitmap of enabled fields.
9968  */
9969 static uint8_t
9970 flow_dv_matcher_enable(uint32_t *match_criteria)
9971 {
9972         uint8_t match_criteria_enable;
9973
9974         match_criteria_enable =
9975                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
9976                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
9977         match_criteria_enable |=
9978                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
9979                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
9980         match_criteria_enable |=
9981                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
9982                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
9983         match_criteria_enable |=
9984                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
9985                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9986         match_criteria_enable |=
9987                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
9988                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
9989         match_criteria_enable |=
9990                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
9991                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
9992         match_criteria_enable |=
9993                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_5)) <<
9994                 MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT;
9995         return match_criteria_enable;
9996 }
9997
9998 static void
9999 __flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria)
10000 {
10001         /*
10002          * Check flow matching criteria first, subtract misc5/4 length if flow
10003          * doesn't own misc5/4 parameters. In some old rdma-core releases,
10004          * misc5/4 are not supported, and matcher creation failure is expected
10005          * w/o subtration. If misc5 is provided, misc4 must be counted in since
10006          * misc5 is right after misc4.
10007          */
10008         if (!(match_criteria & (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT))) {
10009                 *size = MLX5_ST_SZ_BYTES(fte_match_param) -
10010                         MLX5_ST_SZ_BYTES(fte_match_set_misc5);
10011                 if (!(match_criteria & (1 <<
10012                         MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT))) {
10013                         *size -= MLX5_ST_SZ_BYTES(fte_match_set_misc4);
10014                 }
10015         }
10016 }
10017
10018 static struct mlx5_list_entry *
10019 flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
10020                          struct mlx5_list_entry *entry, void *cb_ctx)
10021 {
10022         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10023         struct mlx5_flow_dv_matcher *ref = ctx->data;
10024         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10025                                                             typeof(*tbl), tbl);
10026         struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
10027                                                             sizeof(*resource),
10028                                                             0, SOCKET_ID_ANY);
10029
10030         if (!resource) {
10031                 rte_flow_error_set(ctx->error, ENOMEM,
10032                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10033                                    "cannot create matcher");
10034                 return NULL;
10035         }
10036         memcpy(resource, entry, sizeof(*resource));
10037         resource->tbl = &tbl->tbl;
10038         return &resource->entry;
10039 }
10040
10041 static void
10042 flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
10043                              struct mlx5_list_entry *entry)
10044 {
10045         mlx5_free(entry);
10046 }
10047
10048 struct mlx5_hlist_entry *
10049 flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
10050 {
10051         struct mlx5_dev_ctx_shared *sh = list->ctx;
10052         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10053         struct rte_eth_dev *dev = ctx->dev;
10054         struct mlx5_flow_tbl_data_entry *tbl_data;
10055         struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data;
10056         struct rte_flow_error *error = ctx->error;
10057         union mlx5_flow_tbl_key key = { .v64 = key64 };
10058         struct mlx5_flow_tbl_resource *tbl;
10059         void *domain;
10060         uint32_t idx = 0;
10061         int ret;
10062
10063         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10064         if (!tbl_data) {
10065                 rte_flow_error_set(error, ENOMEM,
10066                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10067                                    NULL,
10068                                    "cannot allocate flow table data entry");
10069                 return NULL;
10070         }
10071         tbl_data->idx = idx;
10072         tbl_data->tunnel = tt_prm->tunnel;
10073         tbl_data->group_id = tt_prm->group_id;
10074         tbl_data->external = !!tt_prm->external;
10075         tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
10076         tbl_data->is_egress = !!key.is_egress;
10077         tbl_data->is_transfer = !!key.is_fdb;
10078         tbl_data->dummy = !!key.dummy;
10079         tbl_data->level = key.level;
10080         tbl_data->id = key.id;
10081         tbl = &tbl_data->tbl;
10082         if (key.dummy)
10083                 return &tbl_data->entry;
10084         if (key.is_fdb)
10085                 domain = sh->fdb_domain;
10086         else if (key.is_egress)
10087                 domain = sh->tx_domain;
10088         else
10089                 domain = sh->rx_domain;
10090         ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
10091         if (ret) {
10092                 rte_flow_error_set(error, ENOMEM,
10093                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10094                                    NULL, "cannot create flow table object");
10095                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10096                 return NULL;
10097         }
10098         if (key.level != 0) {
10099                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
10100                                         (tbl->obj, &tbl_data->jump.action);
10101                 if (ret) {
10102                         rte_flow_error_set(error, ENOMEM,
10103                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10104                                            NULL,
10105                                            "cannot create flow jump action");
10106                         mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10107                         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10108                         return NULL;
10109                 }
10110         }
10111         MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
10112               key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
10113               key.level, key.id);
10114         tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
10115                                               flow_dv_matcher_create_cb,
10116                                               flow_dv_matcher_match_cb,
10117                                               flow_dv_matcher_remove_cb,
10118                                               flow_dv_matcher_clone_cb,
10119                                               flow_dv_matcher_clone_free_cb);
10120         if (!tbl_data->matchers) {
10121                 rte_flow_error_set(error, ENOMEM,
10122                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10123                                    NULL,
10124                                    "cannot create tbl matcher list");
10125                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10126                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10127                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10128                 return NULL;
10129         }
10130         return &tbl_data->entry;
10131 }
10132
10133 int
10134 flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
10135                      struct mlx5_hlist_entry *entry, uint64_t key64,
10136                      void *cb_ctx __rte_unused)
10137 {
10138         struct mlx5_flow_tbl_data_entry *tbl_data =
10139                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10140         union mlx5_flow_tbl_key key = { .v64 = key64 };
10141
10142         return tbl_data->level != key.level ||
10143                tbl_data->id != key.id ||
10144                tbl_data->dummy != key.dummy ||
10145                tbl_data->is_transfer != !!key.is_fdb ||
10146                tbl_data->is_egress != !!key.is_egress;
10147 }
10148
10149 /**
10150  * Get a flow table.
10151  *
10152  * @param[in, out] dev
10153  *   Pointer to rte_eth_dev structure.
10154  * @param[in] table_level
10155  *   Table level to use.
10156  * @param[in] egress
10157  *   Direction of the table.
10158  * @param[in] transfer
10159  *   E-Switch or NIC flow.
10160  * @param[in] dummy
10161  *   Dummy entry for dv API.
10162  * @param[in] table_id
10163  *   Table id to use.
10164  * @param[out] error
10165  *   pointer to error structure.
10166  *
10167  * @return
10168  *   Returns tables resource based on the index, NULL in case of failed.
10169  */
10170 struct mlx5_flow_tbl_resource *
10171 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
10172                          uint32_t table_level, uint8_t egress,
10173                          uint8_t transfer,
10174                          bool external,
10175                          const struct mlx5_flow_tunnel *tunnel,
10176                          uint32_t group_id, uint8_t dummy,
10177                          uint32_t table_id,
10178                          struct rte_flow_error *error)
10179 {
10180         struct mlx5_priv *priv = dev->data->dev_private;
10181         union mlx5_flow_tbl_key table_key = {
10182                 {
10183                         .level = table_level,
10184                         .id = table_id,
10185                         .reserved = 0,
10186                         .dummy = !!dummy,
10187                         .is_fdb = !!transfer,
10188                         .is_egress = !!egress,
10189                 }
10190         };
10191         struct mlx5_flow_tbl_tunnel_prm tt_prm = {
10192                 .tunnel = tunnel,
10193                 .group_id = group_id,
10194                 .external = external,
10195         };
10196         struct mlx5_flow_cb_ctx ctx = {
10197                 .dev = dev,
10198                 .error = error,
10199                 .data = &tt_prm,
10200         };
10201         struct mlx5_hlist_entry *entry;
10202         struct mlx5_flow_tbl_data_entry *tbl_data;
10203
10204         entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
10205         if (!entry) {
10206                 rte_flow_error_set(error, ENOMEM,
10207                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10208                                    "cannot get table");
10209                 return NULL;
10210         }
10211         DRV_LOG(DEBUG, "table_level %u table_id %u "
10212                 "tunnel %u group %u registered.",
10213                 table_level, table_id,
10214                 tunnel ? tunnel->tunnel_id : 0, group_id);
10215         tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10216         return &tbl_data->tbl;
10217 }
10218
10219 void
10220 flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
10221                       struct mlx5_hlist_entry *entry)
10222 {
10223         struct mlx5_dev_ctx_shared *sh = list->ctx;
10224         struct mlx5_flow_tbl_data_entry *tbl_data =
10225                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10226
10227         MLX5_ASSERT(entry && sh);
10228         if (tbl_data->jump.action)
10229                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10230         if (tbl_data->tbl.obj)
10231                 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
10232         if (tbl_data->tunnel_offload && tbl_data->external) {
10233                 struct mlx5_hlist_entry *he;
10234                 struct mlx5_hlist *tunnel_grp_hash;
10235                 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10236                 union tunnel_tbl_key tunnel_key = {
10237                         .tunnel_id = tbl_data->tunnel ?
10238                                         tbl_data->tunnel->tunnel_id : 0,
10239                         .group = tbl_data->group_id
10240                 };
10241                 uint32_t table_level = tbl_data->level;
10242
10243                 tunnel_grp_hash = tbl_data->tunnel ?
10244                                         tbl_data->tunnel->groups :
10245                                         thub->groups;
10246                 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
10247                 if (he)
10248                         mlx5_hlist_unregister(tunnel_grp_hash, he);
10249                 DRV_LOG(DEBUG,
10250                         "table_level %u id %u tunnel %u group %u released.",
10251                         table_level,
10252                         tbl_data->id,
10253                         tbl_data->tunnel ?
10254                         tbl_data->tunnel->tunnel_id : 0,
10255                         tbl_data->group_id);
10256         }
10257         mlx5_list_destroy(tbl_data->matchers);
10258         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10259 }
10260
10261 /**
10262  * Release a flow table.
10263  *
10264  * @param[in] sh
10265  *   Pointer to device shared structure.
10266  * @param[in] tbl
10267  *   Table resource to be released.
10268  *
10269  * @return
10270  *   Returns 0 if table was released, else return 1;
10271  */
10272 static int
10273 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
10274                              struct mlx5_flow_tbl_resource *tbl)
10275 {
10276         struct mlx5_flow_tbl_data_entry *tbl_data =
10277                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10278
10279         if (!tbl)
10280                 return 0;
10281         return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
10282 }
10283
10284 int
10285 flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
10286                          struct mlx5_list_entry *entry, void *cb_ctx)
10287 {
10288         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10289         struct mlx5_flow_dv_matcher *ref = ctx->data;
10290         struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur),
10291                                                         entry);
10292
10293         return cur->crc != ref->crc ||
10294                cur->priority != ref->priority ||
10295                memcmp((const void *)cur->mask.buf,
10296                       (const void *)ref->mask.buf, ref->mask.size);
10297 }
10298
10299 struct mlx5_list_entry *
10300 flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
10301 {
10302         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10303         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10304         struct mlx5_flow_dv_matcher *ref = ctx->data;
10305         struct mlx5_flow_dv_matcher *resource;
10306         struct mlx5dv_flow_matcher_attr dv_attr = {
10307                 .type = IBV_FLOW_ATTR_NORMAL,
10308                 .match_mask = (void *)&ref->mask,
10309         };
10310         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10311                                                             typeof(*tbl), tbl);
10312         int ret;
10313
10314         resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
10315                                SOCKET_ID_ANY);
10316         if (!resource) {
10317                 rte_flow_error_set(ctx->error, ENOMEM,
10318                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10319                                    "cannot create matcher");
10320                 return NULL;
10321         }
10322         *resource = *ref;
10323         dv_attr.match_criteria_enable =
10324                 flow_dv_matcher_enable(resource->mask.buf);
10325         __flow_dv_adjust_buf_size(&ref->mask.size,
10326                                   dv_attr.match_criteria_enable);
10327         dv_attr.priority = ref->priority;
10328         if (tbl->is_egress)
10329                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
10330         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
10331                                                &resource->matcher_object);
10332         if (ret) {
10333                 mlx5_free(resource);
10334                 rte_flow_error_set(ctx->error, ENOMEM,
10335                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10336                                    "cannot create matcher");
10337                 return NULL;
10338         }
10339         return &resource->entry;
10340 }
10341
10342 /**
10343  * Register the flow matcher.
10344  *
10345  * @param[in, out] dev
10346  *   Pointer to rte_eth_dev structure.
10347  * @param[in, out] matcher
10348  *   Pointer to flow matcher.
10349  * @param[in, out] key
10350  *   Pointer to flow table key.
10351  * @parm[in, out] dev_flow
10352  *   Pointer to the dev_flow.
10353  * @param[out] error
10354  *   pointer to error structure.
10355  *
10356  * @return
10357  *   0 on success otherwise -errno and errno is set.
10358  */
10359 static int
10360 flow_dv_matcher_register(struct rte_eth_dev *dev,
10361                          struct mlx5_flow_dv_matcher *ref,
10362                          union mlx5_flow_tbl_key *key,
10363                          struct mlx5_flow *dev_flow,
10364                          const struct mlx5_flow_tunnel *tunnel,
10365                          uint32_t group_id,
10366                          struct rte_flow_error *error)
10367 {
10368         struct mlx5_list_entry *entry;
10369         struct mlx5_flow_dv_matcher *resource;
10370         struct mlx5_flow_tbl_resource *tbl;
10371         struct mlx5_flow_tbl_data_entry *tbl_data;
10372         struct mlx5_flow_cb_ctx ctx = {
10373                 .error = error,
10374                 .data = ref,
10375         };
10376         /**
10377          * tunnel offload API requires this registration for cases when
10378          * tunnel match rule was inserted before tunnel set rule.
10379          */
10380         tbl = flow_dv_tbl_resource_get(dev, key->level,
10381                                        key->is_egress, key->is_fdb,
10382                                        dev_flow->external, tunnel,
10383                                        group_id, 0, key->id, error);
10384         if (!tbl)
10385                 return -rte_errno;      /* No need to refill the error info */
10386         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10387         ref->tbl = tbl;
10388         entry = mlx5_list_register(tbl_data->matchers, &ctx);
10389         if (!entry) {
10390                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
10391                 return rte_flow_error_set(error, ENOMEM,
10392                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10393                                           "cannot allocate ref memory");
10394         }
10395         resource = container_of(entry, typeof(*resource), entry);
10396         dev_flow->handle->dvh.matcher = resource;
10397         return 0;
10398 }
10399
10400 struct mlx5_hlist_entry *
10401 flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
10402 {
10403         struct mlx5_dev_ctx_shared *sh = list->ctx;
10404         struct rte_flow_error *error = ctx;
10405         struct mlx5_flow_dv_tag_resource *entry;
10406         uint32_t idx = 0;
10407         int ret;
10408
10409         entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10410         if (!entry) {
10411                 rte_flow_error_set(error, ENOMEM,
10412                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10413                                    "cannot allocate resource memory");
10414                 return NULL;
10415         }
10416         entry->idx = idx;
10417         entry->tag_id = key;
10418         ret = mlx5_flow_os_create_flow_action_tag(key,
10419                                                   &entry->action);
10420         if (ret) {
10421                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
10422                 rte_flow_error_set(error, ENOMEM,
10423                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10424                                    NULL, "cannot create action");
10425                 return NULL;
10426         }
10427         return &entry->entry;
10428 }
10429
10430 int
10431 flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused,
10432                      struct mlx5_hlist_entry *entry, uint64_t key,
10433                      void *cb_ctx __rte_unused)
10434 {
10435         struct mlx5_flow_dv_tag_resource *tag =
10436                 container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10437
10438         return key != tag->tag_id;
10439 }
10440
10441 /**
10442  * Find existing tag resource or create and register a new one.
10443  *
10444  * @param dev[in, out]
10445  *   Pointer to rte_eth_dev structure.
10446  * @param[in, out] tag_be24
10447  *   Tag value in big endian then R-shift 8.
10448  * @parm[in, out] dev_flow
10449  *   Pointer to the dev_flow.
10450  * @param[out] error
10451  *   pointer to error structure.
10452  *
10453  * @return
10454  *   0 on success otherwise -errno and errno is set.
10455  */
10456 static int
10457 flow_dv_tag_resource_register
10458                         (struct rte_eth_dev *dev,
10459                          uint32_t tag_be24,
10460                          struct mlx5_flow *dev_flow,
10461                          struct rte_flow_error *error)
10462 {
10463         struct mlx5_priv *priv = dev->data->dev_private;
10464         struct mlx5_flow_dv_tag_resource *resource;
10465         struct mlx5_hlist_entry *entry;
10466
10467         entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
10468         if (entry) {
10469                 resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
10470                                         entry);
10471                 dev_flow->handle->dvh.rix_tag = resource->idx;
10472                 dev_flow->dv.tag_resource = resource;
10473                 return 0;
10474         }
10475         return -rte_errno;
10476 }
10477
10478 void
10479 flow_dv_tag_remove_cb(struct mlx5_hlist *list,
10480                       struct mlx5_hlist_entry *entry)
10481 {
10482         struct mlx5_dev_ctx_shared *sh = list->ctx;
10483         struct mlx5_flow_dv_tag_resource *tag =
10484                 container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10485
10486         MLX5_ASSERT(tag && sh && tag->action);
10487         claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
10488         DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag);
10489         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10490 }
10491
10492 /**
10493  * Release the tag.
10494  *
10495  * @param dev
10496  *   Pointer to Ethernet device.
10497  * @param tag_idx
10498  *   Tag index.
10499  *
10500  * @return
10501  *   1 while a reference on it exists, 0 when freed.
10502  */
10503 static int
10504 flow_dv_tag_release(struct rte_eth_dev *dev,
10505                     uint32_t tag_idx)
10506 {
10507         struct mlx5_priv *priv = dev->data->dev_private;
10508         struct mlx5_flow_dv_tag_resource *tag;
10509
10510         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
10511         if (!tag)
10512                 return 0;
10513         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
10514                 dev->data->port_id, (void *)tag, tag->entry.ref_cnt);
10515         return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry);
10516 }
10517
10518 /**
10519  * Translate port ID action to vport.
10520  *
10521  * @param[in] dev
10522  *   Pointer to rte_eth_dev structure.
10523  * @param[in] action
10524  *   Pointer to the port ID action.
10525  * @param[out] dst_port_id
10526  *   The target port ID.
10527  * @param[out] error
10528  *   Pointer to the error structure.
10529  *
10530  * @return
10531  *   0 on success, a negative errno value otherwise and rte_errno is set.
10532  */
10533 static int
10534 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
10535                                  const struct rte_flow_action *action,
10536                                  uint32_t *dst_port_id,
10537                                  struct rte_flow_error *error)
10538 {
10539         uint32_t port;
10540         struct mlx5_priv *priv;
10541         const struct rte_flow_action_port_id *conf =
10542                         (const struct rte_flow_action_port_id *)action->conf;
10543
10544         port = conf->original ? dev->data->port_id : conf->id;
10545         priv = mlx5_port_to_eswitch_info(port, false);
10546         if (!priv)
10547                 return rte_flow_error_set(error, -rte_errno,
10548                                           RTE_FLOW_ERROR_TYPE_ACTION,
10549                                           NULL,
10550                                           "No eswitch info was found for port");
10551 #ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT
10552         /*
10553          * This parameter is transferred to
10554          * mlx5dv_dr_action_create_dest_ib_port().
10555          */
10556         *dst_port_id = priv->dev_port;
10557 #else
10558         /*
10559          * Legacy mode, no LAG configurations is supported.
10560          * This parameter is transferred to
10561          * mlx5dv_dr_action_create_dest_vport().
10562          */
10563         *dst_port_id = priv->vport_id;
10564 #endif
10565         return 0;
10566 }
10567
10568 /**
10569  * Create a counter with aging configuration.
10570  *
10571  * @param[in] dev
10572  *   Pointer to rte_eth_dev structure.
10573  * @param[in] dev_flow
10574  *   Pointer to the mlx5_flow.
10575  * @param[out] count
10576  *   Pointer to the counter action configuration.
10577  * @param[in] age
10578  *   Pointer to the aging action configuration.
10579  *
10580  * @return
10581  *   Index to flow counter on success, 0 otherwise.
10582  */
10583 static uint32_t
10584 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
10585                                 struct mlx5_flow *dev_flow,
10586                                 const struct rte_flow_action_count *count,
10587                                 const struct rte_flow_action_age *age)
10588 {
10589         uint32_t counter;
10590         struct mlx5_age_param *age_param;
10591
10592         if (count && count->shared)
10593                 counter = flow_dv_counter_get_shared(dev, count->id);
10594         else
10595                 counter = flow_dv_counter_alloc(dev, !!age);
10596         if (!counter || age == NULL)
10597                 return counter;
10598         age_param = flow_dv_counter_idx_get_age(dev, counter);
10599         age_param->context = age->context ? age->context :
10600                 (void *)(uintptr_t)(dev_flow->flow_idx);
10601         age_param->timeout = age->timeout;
10602         age_param->port_id = dev->data->port_id;
10603         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
10604         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
10605         return counter;
10606 }
10607
10608 /**
10609  * Add Tx queue matcher
10610  *
10611  * @param[in] dev
10612  *   Pointer to the dev struct.
10613  * @param[in, out] matcher
10614  *   Flow matcher.
10615  * @param[in, out] key
10616  *   Flow matcher value.
10617  * @param[in] item
10618  *   Flow pattern to translate.
10619  * @param[in] inner
10620  *   Item is inner pattern.
10621  */
10622 static void
10623 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
10624                                 void *matcher, void *key,
10625                                 const struct rte_flow_item *item)
10626 {
10627         const struct mlx5_rte_flow_item_tx_queue *queue_m;
10628         const struct mlx5_rte_flow_item_tx_queue *queue_v;
10629         void *misc_m =
10630                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
10631         void *misc_v =
10632                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
10633         struct mlx5_txq_ctrl *txq;
10634         uint32_t queue;
10635
10636
10637         queue_m = (const void *)item->mask;
10638         if (!queue_m)
10639                 return;
10640         queue_v = (const void *)item->spec;
10641         if (!queue_v)
10642                 return;
10643         txq = mlx5_txq_get(dev, queue_v->queue);
10644         if (!txq)
10645                 return;
10646         queue = txq->obj->sq->id;
10647         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
10648         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
10649                  queue & queue_m->queue);
10650         mlx5_txq_release(dev, queue_v->queue);
10651 }
10652
10653 /**
10654  * Set the hash fields according to the @p flow information.
10655  *
10656  * @param[in] dev_flow
10657  *   Pointer to the mlx5_flow.
10658  * @param[in] rss_desc
10659  *   Pointer to the mlx5_flow_rss_desc.
10660  */
10661 static void
10662 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
10663                        struct mlx5_flow_rss_desc *rss_desc)
10664 {
10665         uint64_t items = dev_flow->handle->layers;
10666         int rss_inner = 0;
10667         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
10668
10669         dev_flow->hash_fields = 0;
10670 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
10671         if (rss_desc->level >= 2) {
10672                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
10673                 rss_inner = 1;
10674         }
10675 #endif
10676         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
10677             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
10678                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
10679                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10680                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
10681                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10682                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
10683                         else
10684                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
10685                 }
10686         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
10687                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
10688                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
10689                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10690                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
10691                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10692                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
10693                         else
10694                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
10695                 }
10696         }
10697         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
10698             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
10699                 if (rss_types & ETH_RSS_UDP) {
10700                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10701                                 dev_flow->hash_fields |=
10702                                                 IBV_RX_HASH_SRC_PORT_UDP;
10703                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10704                                 dev_flow->hash_fields |=
10705                                                 IBV_RX_HASH_DST_PORT_UDP;
10706                         else
10707                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
10708                 }
10709         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
10710                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
10711                 if (rss_types & ETH_RSS_TCP) {
10712                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10713                                 dev_flow->hash_fields |=
10714                                                 IBV_RX_HASH_SRC_PORT_TCP;
10715                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10716                                 dev_flow->hash_fields |=
10717                                                 IBV_RX_HASH_DST_PORT_TCP;
10718                         else
10719                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
10720                 }
10721         }
10722 }
10723
10724 /**
10725  * Prepare an Rx Hash queue.
10726  *
10727  * @param dev
10728  *   Pointer to Ethernet device.
10729  * @param[in] dev_flow
10730  *   Pointer to the mlx5_flow.
10731  * @param[in] rss_desc
10732  *   Pointer to the mlx5_flow_rss_desc.
10733  * @param[out] hrxq_idx
10734  *   Hash Rx queue index.
10735  *
10736  * @return
10737  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
10738  */
10739 static struct mlx5_hrxq *
10740 flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
10741                      struct mlx5_flow *dev_flow,
10742                      struct mlx5_flow_rss_desc *rss_desc,
10743                      uint32_t *hrxq_idx)
10744 {
10745         struct mlx5_priv *priv = dev->data->dev_private;
10746         struct mlx5_flow_handle *dh = dev_flow->handle;
10747         struct mlx5_hrxq *hrxq;
10748
10749         MLX5_ASSERT(rss_desc->queue_num);
10750         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
10751         rss_desc->hash_fields = dev_flow->hash_fields;
10752         rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL);
10753         rss_desc->shared_rss = 0;
10754         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
10755         if (!*hrxq_idx)
10756                 return NULL;
10757         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10758                               *hrxq_idx);
10759         return hrxq;
10760 }
10761
10762 /**
10763  * Release sample sub action resource.
10764  *
10765  * @param[in, out] dev
10766  *   Pointer to rte_eth_dev structure.
10767  * @param[in] act_res
10768  *   Pointer to sample sub action resource.
10769  */
10770 static void
10771 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
10772                                    struct mlx5_flow_sub_actions_idx *act_res)
10773 {
10774         if (act_res->rix_hrxq) {
10775                 mlx5_hrxq_release(dev, act_res->rix_hrxq);
10776                 act_res->rix_hrxq = 0;
10777         }
10778         if (act_res->rix_encap_decap) {
10779                 flow_dv_encap_decap_resource_release(dev,
10780                                                      act_res->rix_encap_decap);
10781                 act_res->rix_encap_decap = 0;
10782         }
10783         if (act_res->rix_port_id_action) {
10784                 flow_dv_port_id_action_resource_release(dev,
10785                                                 act_res->rix_port_id_action);
10786                 act_res->rix_port_id_action = 0;
10787         }
10788         if (act_res->rix_tag) {
10789                 flow_dv_tag_release(dev, act_res->rix_tag);
10790                 act_res->rix_tag = 0;
10791         }
10792         if (act_res->rix_jump) {
10793                 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump);
10794                 act_res->rix_jump = 0;
10795         }
10796 }
10797
10798 int
10799 flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
10800                         struct mlx5_list_entry *entry, void *cb_ctx)
10801 {
10802         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10803         struct rte_eth_dev *dev = ctx->dev;
10804         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
10805         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
10806                                                               typeof(*resource),
10807                                                               entry);
10808
10809         if (ctx_resource->ratio == resource->ratio &&
10810             ctx_resource->ft_type == resource->ft_type &&
10811             ctx_resource->ft_id == resource->ft_id &&
10812             ctx_resource->set_action == resource->set_action &&
10813             !memcmp((void *)&ctx_resource->sample_act,
10814                     (void *)&resource->sample_act,
10815                     sizeof(struct mlx5_flow_sub_actions_list))) {
10816                 /*
10817                  * Existing sample action should release the prepared
10818                  * sub-actions reference counter.
10819                  */
10820                 flow_dv_sample_sub_actions_release(dev,
10821                                                    &ctx_resource->sample_idx);
10822                 return 0;
10823         }
10824         return 1;
10825 }
10826
10827 struct mlx5_list_entry *
10828 flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
10829 {
10830         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10831         struct rte_eth_dev *dev = ctx->dev;
10832         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
10833         void **sample_dv_actions = ctx_resource->sub_actions;
10834         struct mlx5_flow_dv_sample_resource *resource;
10835         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
10836         struct mlx5_priv *priv = dev->data->dev_private;
10837         struct mlx5_dev_ctx_shared *sh = priv->sh;
10838         struct mlx5_flow_tbl_resource *tbl;
10839         uint32_t idx = 0;
10840         const uint32_t next_ft_step = 1;
10841         uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
10842         uint8_t is_egress = 0;
10843         uint8_t is_transfer = 0;
10844         struct rte_flow_error *error = ctx->error;
10845
10846         /* Register new sample resource. */
10847         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
10848         if (!resource) {
10849                 rte_flow_error_set(error, ENOMEM,
10850                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10851                                           NULL,
10852                                           "cannot allocate resource memory");
10853                 return NULL;
10854         }
10855         *resource = *ctx_resource;
10856         /* Create normal path table level */
10857         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
10858                 is_transfer = 1;
10859         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
10860                 is_egress = 1;
10861         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
10862                                         is_egress, is_transfer,
10863                                         true, NULL, 0, 0, 0, error);
10864         if (!tbl) {
10865                 rte_flow_error_set(error, ENOMEM,
10866                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10867                                           NULL,
10868                                           "fail to create normal path table "
10869                                           "for sample");
10870                 goto error;
10871         }
10872         resource->normal_path_tbl = tbl;
10873         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10874                 if (!sh->default_miss_action) {
10875                         rte_flow_error_set(error, ENOMEM,
10876                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10877                                                 NULL,
10878                                                 "default miss action was not "
10879                                                 "created");
10880                         goto error;
10881                 }
10882                 sample_dv_actions[ctx_resource->sample_act.actions_num++] =
10883                                                 sh->default_miss_action;
10884         }
10885         /* Create a DR sample action */
10886         sampler_attr.sample_ratio = resource->ratio;
10887         sampler_attr.default_next_table = tbl->obj;
10888         sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
10889         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
10890                                                         &sample_dv_actions[0];
10891         sampler_attr.action = resource->set_action;
10892         if (mlx5_os_flow_dr_create_flow_action_sampler
10893                         (&sampler_attr, &resource->verbs_action)) {
10894                 rte_flow_error_set(error, ENOMEM,
10895                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10896                                         NULL, "cannot create sample action");
10897                 goto error;
10898         }
10899         resource->idx = idx;
10900         resource->dev = dev;
10901         return &resource->entry;
10902 error:
10903         if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
10904                 flow_dv_sample_sub_actions_release(dev,
10905                                                    &resource->sample_idx);
10906         if (resource->normal_path_tbl)
10907                 flow_dv_tbl_resource_release(MLX5_SH(dev),
10908                                 resource->normal_path_tbl);
10909         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10910         return NULL;
10911
10912 }
10913
10914 struct mlx5_list_entry *
10915 flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
10916                          struct mlx5_list_entry *entry __rte_unused,
10917                          void *cb_ctx)
10918 {
10919         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10920         struct rte_eth_dev *dev = ctx->dev;
10921         struct mlx5_flow_dv_sample_resource *resource;
10922         struct mlx5_priv *priv = dev->data->dev_private;
10923         struct mlx5_dev_ctx_shared *sh = priv->sh;
10924         uint32_t idx = 0;
10925
10926         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
10927         if (!resource) {
10928                 rte_flow_error_set(ctx->error, ENOMEM,
10929                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10930                                           NULL,
10931                                           "cannot allocate resource memory");
10932                 return NULL;
10933         }
10934         memcpy(resource, entry, sizeof(*resource));
10935         resource->idx = idx;
10936         resource->dev = dev;
10937         return &resource->entry;
10938 }
10939
10940 void
10941 flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
10942                              struct mlx5_list_entry *entry)
10943 {
10944         struct mlx5_flow_dv_sample_resource *resource =
10945                                   container_of(entry, typeof(*resource), entry);
10946         struct rte_eth_dev *dev = resource->dev;
10947         struct mlx5_priv *priv = dev->data->dev_private;
10948
10949         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
10950 }
10951
10952 /**
10953  * Find existing sample resource or create and register a new one.
10954  *
10955  * @param[in, out] dev
10956  *   Pointer to rte_eth_dev structure.
10957  * @param[in] ref
10958  *   Pointer to sample resource reference.
10959  * @parm[in, out] dev_flow
10960  *   Pointer to the dev_flow.
10961  * @param[out] error
10962  *   pointer to error structure.
10963  *
10964  * @return
10965  *   0 on success otherwise -errno and errno is set.
10966  */
10967 static int
10968 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
10969                          struct mlx5_flow_dv_sample_resource *ref,
10970                          struct mlx5_flow *dev_flow,
10971                          struct rte_flow_error *error)
10972 {
10973         struct mlx5_flow_dv_sample_resource *resource;
10974         struct mlx5_list_entry *entry;
10975         struct mlx5_priv *priv = dev->data->dev_private;
10976         struct mlx5_flow_cb_ctx ctx = {
10977                 .dev = dev,
10978                 .error = error,
10979                 .data = ref,
10980         };
10981
10982         entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
10983         if (!entry)
10984                 return -rte_errno;
10985         resource = container_of(entry, typeof(*resource), entry);
10986         dev_flow->handle->dvh.rix_sample = resource->idx;
10987         dev_flow->dv.sample_res = resource;
10988         return 0;
10989 }
10990
10991 int
10992 flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
10993                             struct mlx5_list_entry *entry, void *cb_ctx)
10994 {
10995         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10996         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
10997         struct rte_eth_dev *dev = ctx->dev;
10998         struct mlx5_flow_dv_dest_array_resource *resource =
10999                                   container_of(entry, typeof(*resource), entry);
11000         uint32_t idx = 0;
11001
11002         if (ctx_resource->num_of_dest == resource->num_of_dest &&
11003             ctx_resource->ft_type == resource->ft_type &&
11004             !memcmp((void *)resource->sample_act,
11005                     (void *)ctx_resource->sample_act,
11006                    (ctx_resource->num_of_dest *
11007                    sizeof(struct mlx5_flow_sub_actions_list)))) {
11008                 /*
11009                  * Existing sample action should release the prepared
11010                  * sub-actions reference counter.
11011                  */
11012                 for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11013                         flow_dv_sample_sub_actions_release(dev,
11014                                         &ctx_resource->sample_idx[idx]);
11015                 return 0;
11016         }
11017         return 1;
11018 }
11019
11020 struct mlx5_list_entry *
11021 flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11022 {
11023         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11024         struct rte_eth_dev *dev = ctx->dev;
11025         struct mlx5_flow_dv_dest_array_resource *resource;
11026         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11027         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
11028         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
11029         struct mlx5_priv *priv = dev->data->dev_private;
11030         struct mlx5_dev_ctx_shared *sh = priv->sh;
11031         struct mlx5_flow_sub_actions_list *sample_act;
11032         struct mlx5dv_dr_domain *domain;
11033         uint32_t idx = 0, res_idx = 0;
11034         struct rte_flow_error *error = ctx->error;
11035         uint64_t action_flags;
11036         int ret;
11037
11038         /* Register new destination array resource. */
11039         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11040                                             &res_idx);
11041         if (!resource) {
11042                 rte_flow_error_set(error, ENOMEM,
11043                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11044                                           NULL,
11045                                           "cannot allocate resource memory");
11046                 return NULL;
11047         }
11048         *resource = *ctx_resource;
11049         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11050                 domain = sh->fdb_domain;
11051         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
11052                 domain = sh->rx_domain;
11053         else
11054                 domain = sh->tx_domain;
11055         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11056                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
11057                                  mlx5_malloc(MLX5_MEM_ZERO,
11058                                  sizeof(struct mlx5dv_dr_action_dest_attr),
11059                                  0, SOCKET_ID_ANY);
11060                 if (!dest_attr[idx]) {
11061                         rte_flow_error_set(error, ENOMEM,
11062                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11063                                            NULL,
11064                                            "cannot allocate resource memory");
11065                         goto error;
11066                 }
11067                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
11068                 sample_act = &ctx_resource->sample_act[idx];
11069                 action_flags = sample_act->action_flags;
11070                 switch (action_flags) {
11071                 case MLX5_FLOW_ACTION_QUEUE:
11072                         dest_attr[idx]->dest = sample_act->dr_queue_action;
11073                         break;
11074                 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP):
11075                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
11076                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
11077                         dest_attr[idx]->dest_reformat->reformat =
11078                                         sample_act->dr_encap_action;
11079                         dest_attr[idx]->dest_reformat->dest =
11080                                         sample_act->dr_port_id_action;
11081                         break;
11082                 case MLX5_FLOW_ACTION_PORT_ID:
11083                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
11084                         break;
11085                 case MLX5_FLOW_ACTION_JUMP:
11086                         dest_attr[idx]->dest = sample_act->dr_jump_action;
11087                         break;
11088                 default:
11089                         rte_flow_error_set(error, EINVAL,
11090                                            RTE_FLOW_ERROR_TYPE_ACTION,
11091                                            NULL,
11092                                            "unsupported actions type");
11093                         goto error;
11094                 }
11095         }
11096         /* create a dest array actioin */
11097         ret = mlx5_os_flow_dr_create_flow_action_dest_array
11098                                                 (domain,
11099                                                  resource->num_of_dest,
11100                                                  dest_attr,
11101                                                  &resource->action);
11102         if (ret) {
11103                 rte_flow_error_set(error, ENOMEM,
11104                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11105                                    NULL,
11106                                    "cannot create destination array action");
11107                 goto error;
11108         }
11109         resource->idx = res_idx;
11110         resource->dev = dev;
11111         for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11112                 mlx5_free(dest_attr[idx]);
11113         return &resource->entry;
11114 error:
11115         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11116                 flow_dv_sample_sub_actions_release(dev,
11117                                                    &resource->sample_idx[idx]);
11118                 if (dest_attr[idx])
11119                         mlx5_free(dest_attr[idx]);
11120         }
11121         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
11122         return NULL;
11123 }
11124
11125 struct mlx5_list_entry *
11126 flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
11127                             struct mlx5_list_entry *entry __rte_unused,
11128                             void *cb_ctx)
11129 {
11130         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11131         struct rte_eth_dev *dev = ctx->dev;
11132         struct mlx5_flow_dv_dest_array_resource *resource;
11133         struct mlx5_priv *priv = dev->data->dev_private;
11134         struct mlx5_dev_ctx_shared *sh = priv->sh;
11135         uint32_t res_idx = 0;
11136         struct rte_flow_error *error = ctx->error;
11137
11138         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11139                                       &res_idx);
11140         if (!resource) {
11141                 rte_flow_error_set(error, ENOMEM,
11142                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11143                                           NULL,
11144                                           "cannot allocate dest-array memory");
11145                 return NULL;
11146         }
11147         memcpy(resource, entry, sizeof(*resource));
11148         resource->idx = res_idx;
11149         resource->dev = dev;
11150         return &resource->entry;
11151 }
11152
11153 void
11154 flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
11155                                  struct mlx5_list_entry *entry)
11156 {
11157         struct mlx5_flow_dv_dest_array_resource *resource =
11158                         container_of(entry, typeof(*resource), entry);
11159         struct rte_eth_dev *dev = resource->dev;
11160         struct mlx5_priv *priv = dev->data->dev_private;
11161
11162         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
11163 }
11164
11165 /**
11166  * Find existing destination array resource or create and register a new one.
11167  *
11168  * @param[in, out] dev
11169  *   Pointer to rte_eth_dev structure.
11170  * @param[in] ref
11171  *   Pointer to destination array resource reference.
11172  * @parm[in, out] dev_flow
11173  *   Pointer to the dev_flow.
11174  * @param[out] error
11175  *   pointer to error structure.
11176  *
11177  * @return
11178  *   0 on success otherwise -errno and errno is set.
11179  */
11180 static int
11181 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
11182                          struct mlx5_flow_dv_dest_array_resource *ref,
11183                          struct mlx5_flow *dev_flow,
11184                          struct rte_flow_error *error)
11185 {
11186         struct mlx5_flow_dv_dest_array_resource *resource;
11187         struct mlx5_priv *priv = dev->data->dev_private;
11188         struct mlx5_list_entry *entry;
11189         struct mlx5_flow_cb_ctx ctx = {
11190                 .dev = dev,
11191                 .error = error,
11192                 .data = ref,
11193         };
11194
11195         entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
11196         if (!entry)
11197                 return -rte_errno;
11198         resource = container_of(entry, typeof(*resource), entry);
11199         dev_flow->handle->dvh.rix_dest_array = resource->idx;
11200         dev_flow->dv.dest_array_res = resource;
11201         return 0;
11202 }
11203
11204 /**
11205  * Convert Sample action to DV specification.
11206  *
11207  * @param[in] dev
11208  *   Pointer to rte_eth_dev structure.
11209  * @param[in] action
11210  *   Pointer to sample action structure.
11211  * @param[in, out] dev_flow
11212  *   Pointer to the mlx5_flow.
11213  * @param[in] attr
11214  *   Pointer to the flow attributes.
11215  * @param[in, out] num_of_dest
11216  *   Pointer to the num of destination.
11217  * @param[in, out] sample_actions
11218  *   Pointer to sample actions list.
11219  * @param[in, out] res
11220  *   Pointer to sample resource.
11221  * @param[out] error
11222  *   Pointer to the error structure.
11223  *
11224  * @return
11225  *   0 on success, a negative errno value otherwise and rte_errno is set.
11226  */
11227 static int
11228 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
11229                                 const struct rte_flow_action_sample *action,
11230                                 struct mlx5_flow *dev_flow,
11231                                 const struct rte_flow_attr *attr,
11232                                 uint32_t *num_of_dest,
11233                                 void **sample_actions,
11234                                 struct mlx5_flow_dv_sample_resource *res,
11235                                 struct rte_flow_error *error)
11236 {
11237         struct mlx5_priv *priv = dev->data->dev_private;
11238         const struct rte_flow_action *sub_actions;
11239         struct mlx5_flow_sub_actions_list *sample_act;
11240         struct mlx5_flow_sub_actions_idx *sample_idx;
11241         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11242         struct rte_flow *flow = dev_flow->flow;
11243         struct mlx5_flow_rss_desc *rss_desc;
11244         uint64_t action_flags = 0;
11245
11246         MLX5_ASSERT(wks);
11247         rss_desc = &wks->rss_desc;
11248         sample_act = &res->sample_act;
11249         sample_idx = &res->sample_idx;
11250         res->ratio = action->ratio;
11251         sub_actions = action->actions;
11252         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
11253                 int type = sub_actions->type;
11254                 uint32_t pre_rix = 0;
11255                 void *pre_r;
11256                 switch (type) {
11257                 case RTE_FLOW_ACTION_TYPE_QUEUE:
11258                 {
11259                         const struct rte_flow_action_queue *queue;
11260                         struct mlx5_hrxq *hrxq;
11261                         uint32_t hrxq_idx;
11262
11263                         queue = sub_actions->conf;
11264                         rss_desc->queue_num = 1;
11265                         rss_desc->queue[0] = queue->index;
11266                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11267                                                     rss_desc, &hrxq_idx);
11268                         if (!hrxq)
11269                                 return rte_flow_error_set
11270                                         (error, rte_errno,
11271                                          RTE_FLOW_ERROR_TYPE_ACTION,
11272                                          NULL,
11273                                          "cannot create fate queue");
11274                         sample_act->dr_queue_action = hrxq->action;
11275                         sample_idx->rix_hrxq = hrxq_idx;
11276                         sample_actions[sample_act->actions_num++] =
11277                                                 hrxq->action;
11278                         (*num_of_dest)++;
11279                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
11280                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11281                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11282                         dev_flow->handle->fate_action =
11283                                         MLX5_FLOW_FATE_QUEUE;
11284                         break;
11285                 }
11286                 case RTE_FLOW_ACTION_TYPE_RSS:
11287                 {
11288                         struct mlx5_hrxq *hrxq;
11289                         uint32_t hrxq_idx;
11290                         const struct rte_flow_action_rss *rss;
11291                         const uint8_t *rss_key;
11292
11293                         rss = sub_actions->conf;
11294                         memcpy(rss_desc->queue, rss->queue,
11295                                rss->queue_num * sizeof(uint16_t));
11296                         rss_desc->queue_num = rss->queue_num;
11297                         /* NULL RSS key indicates default RSS key. */
11298                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11299                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11300                         /*
11301                          * rss->level and rss.types should be set in advance
11302                          * when expanding items for RSS.
11303                          */
11304                         flow_dv_hashfields_set(dev_flow, rss_desc);
11305                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11306                                                     rss_desc, &hrxq_idx);
11307                         if (!hrxq)
11308                                 return rte_flow_error_set
11309                                         (error, rte_errno,
11310                                          RTE_FLOW_ERROR_TYPE_ACTION,
11311                                          NULL,
11312                                          "cannot create fate queue");
11313                         sample_act->dr_queue_action = hrxq->action;
11314                         sample_idx->rix_hrxq = hrxq_idx;
11315                         sample_actions[sample_act->actions_num++] =
11316                                                 hrxq->action;
11317                         (*num_of_dest)++;
11318                         action_flags |= MLX5_FLOW_ACTION_RSS;
11319                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11320                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11321                         dev_flow->handle->fate_action =
11322                                         MLX5_FLOW_FATE_QUEUE;
11323                         break;
11324                 }
11325                 case RTE_FLOW_ACTION_TYPE_MARK:
11326                 {
11327                         uint32_t tag_be = mlx5_flow_mark_set
11328                                 (((const struct rte_flow_action_mark *)
11329                                 (sub_actions->conf))->id);
11330
11331                         dev_flow->handle->mark = 1;
11332                         pre_rix = dev_flow->handle->dvh.rix_tag;
11333                         /* Save the mark resource before sample */
11334                         pre_r = dev_flow->dv.tag_resource;
11335                         if (flow_dv_tag_resource_register(dev, tag_be,
11336                                                   dev_flow, error))
11337                                 return -rte_errno;
11338                         MLX5_ASSERT(dev_flow->dv.tag_resource);
11339                         sample_act->dr_tag_action =
11340                                 dev_flow->dv.tag_resource->action;
11341                         sample_idx->rix_tag =
11342                                 dev_flow->handle->dvh.rix_tag;
11343                         sample_actions[sample_act->actions_num++] =
11344                                                 sample_act->dr_tag_action;
11345                         /* Recover the mark resource after sample */
11346                         dev_flow->dv.tag_resource = pre_r;
11347                         dev_flow->handle->dvh.rix_tag = pre_rix;
11348                         action_flags |= MLX5_FLOW_ACTION_MARK;
11349                         break;
11350                 }
11351                 case RTE_FLOW_ACTION_TYPE_COUNT:
11352                 {
11353                         if (!flow->counter) {
11354                                 flow->counter =
11355                                         flow_dv_translate_create_counter(dev,
11356                                                 dev_flow, sub_actions->conf,
11357                                                 0);
11358                                 if (!flow->counter)
11359                                         return rte_flow_error_set
11360                                                 (error, rte_errno,
11361                                                 RTE_FLOW_ERROR_TYPE_ACTION,
11362                                                 NULL,
11363                                                 "cannot create counter"
11364                                                 " object.");
11365                         }
11366                         sample_act->dr_cnt_action =
11367                                   (flow_dv_counter_get_by_idx(dev,
11368                                   flow->counter, NULL))->action;
11369                         sample_actions[sample_act->actions_num++] =
11370                                                 sample_act->dr_cnt_action;
11371                         action_flags |= MLX5_FLOW_ACTION_COUNT;
11372                         break;
11373                 }
11374                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
11375                 {
11376                         struct mlx5_flow_dv_port_id_action_resource
11377                                         port_id_resource;
11378                         uint32_t port_id = 0;
11379
11380                         memset(&port_id_resource, 0, sizeof(port_id_resource));
11381                         /* Save the port id resource before sample */
11382                         pre_rix = dev_flow->handle->rix_port_id_action;
11383                         pre_r = dev_flow->dv.port_id_action;
11384                         if (flow_dv_translate_action_port_id(dev, sub_actions,
11385                                                              &port_id, error))
11386                                 return -rte_errno;
11387                         port_id_resource.port_id = port_id;
11388                         if (flow_dv_port_id_action_resource_register
11389                             (dev, &port_id_resource, dev_flow, error))
11390                                 return -rte_errno;
11391                         sample_act->dr_port_id_action =
11392                                 dev_flow->dv.port_id_action->action;
11393                         sample_idx->rix_port_id_action =
11394                                 dev_flow->handle->rix_port_id_action;
11395                         sample_actions[sample_act->actions_num++] =
11396                                                 sample_act->dr_port_id_action;
11397                         /* Recover the port id resource after sample */
11398                         dev_flow->dv.port_id_action = pre_r;
11399                         dev_flow->handle->rix_port_id_action = pre_rix;
11400                         (*num_of_dest)++;
11401                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
11402                         break;
11403                 }
11404                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
11405                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
11406                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
11407                         /* Save the encap resource before sample */
11408                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
11409                         pre_r = dev_flow->dv.encap_decap;
11410                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
11411                                                            dev_flow,
11412                                                            attr->transfer,
11413                                                            error))
11414                                 return -rte_errno;
11415                         sample_act->dr_encap_action =
11416                                 dev_flow->dv.encap_decap->action;
11417                         sample_idx->rix_encap_decap =
11418                                 dev_flow->handle->dvh.rix_encap_decap;
11419                         sample_actions[sample_act->actions_num++] =
11420                                                 sample_act->dr_encap_action;
11421                         /* Recover the encap resource after sample */
11422                         dev_flow->dv.encap_decap = pre_r;
11423                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
11424                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
11425                         break;
11426                 default:
11427                         return rte_flow_error_set(error, EINVAL,
11428                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11429                                 NULL,
11430                                 "Not support for sampler action");
11431                 }
11432         }
11433         sample_act->action_flags = action_flags;
11434         res->ft_id = dev_flow->dv.group;
11435         if (attr->transfer) {
11436                 union {
11437                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
11438                         uint64_t set_action;
11439                 } action_ctx = { .set_action = 0 };
11440
11441                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
11442                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
11443                          MLX5_MODIFICATION_TYPE_SET);
11444                 MLX5_SET(set_action_in, action_ctx.action_in, field,
11445                          MLX5_MODI_META_REG_C_0);
11446                 MLX5_SET(set_action_in, action_ctx.action_in, data,
11447                          priv->vport_meta_tag);
11448                 res->set_action = action_ctx.set_action;
11449         } else if (attr->ingress) {
11450                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
11451         } else {
11452                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
11453         }
11454         return 0;
11455 }
11456
11457 /**
11458  * Convert Sample action to DV specification.
11459  *
11460  * @param[in] dev
11461  *   Pointer to rte_eth_dev structure.
11462  * @param[in, out] dev_flow
11463  *   Pointer to the mlx5_flow.
11464  * @param[in] num_of_dest
11465  *   The num of destination.
11466  * @param[in, out] res
11467  *   Pointer to sample resource.
11468  * @param[in, out] mdest_res
11469  *   Pointer to destination array resource.
11470  * @param[in] sample_actions
11471  *   Pointer to sample path actions list.
11472  * @param[in] action_flags
11473  *   Holds the actions detected until now.
11474  * @param[out] error
11475  *   Pointer to the error structure.
11476  *
11477  * @return
11478  *   0 on success, a negative errno value otherwise and rte_errno is set.
11479  */
11480 static int
11481 flow_dv_create_action_sample(struct rte_eth_dev *dev,
11482                              struct mlx5_flow *dev_flow,
11483                              uint32_t num_of_dest,
11484                              struct mlx5_flow_dv_sample_resource *res,
11485                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
11486                              void **sample_actions,
11487                              uint64_t action_flags,
11488                              struct rte_flow_error *error)
11489 {
11490         /* update normal path action resource into last index of array */
11491         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
11492         struct mlx5_flow_sub_actions_list *sample_act =
11493                                         &mdest_res->sample_act[dest_index];
11494         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11495         struct mlx5_flow_rss_desc *rss_desc;
11496         uint32_t normal_idx = 0;
11497         struct mlx5_hrxq *hrxq;
11498         uint32_t hrxq_idx;
11499
11500         MLX5_ASSERT(wks);
11501         rss_desc = &wks->rss_desc;
11502         if (num_of_dest > 1) {
11503                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
11504                         /* Handle QP action for mirroring */
11505                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11506                                                     rss_desc, &hrxq_idx);
11507                         if (!hrxq)
11508                                 return rte_flow_error_set
11509                                      (error, rte_errno,
11510                                       RTE_FLOW_ERROR_TYPE_ACTION,
11511                                       NULL,
11512                                       "cannot create rx queue");
11513                         normal_idx++;
11514                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
11515                         sample_act->dr_queue_action = hrxq->action;
11516                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11517                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11518                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
11519                 }
11520                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
11521                         normal_idx++;
11522                         mdest_res->sample_idx[dest_index].rix_encap_decap =
11523                                 dev_flow->handle->dvh.rix_encap_decap;
11524                         sample_act->dr_encap_action =
11525                                 dev_flow->dv.encap_decap->action;
11526                         dev_flow->handle->dvh.rix_encap_decap = 0;
11527                 }
11528                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
11529                         normal_idx++;
11530                         mdest_res->sample_idx[dest_index].rix_port_id_action =
11531                                 dev_flow->handle->rix_port_id_action;
11532                         sample_act->dr_port_id_action =
11533                                 dev_flow->dv.port_id_action->action;
11534                         dev_flow->handle->rix_port_id_action = 0;
11535                 }
11536                 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) {
11537                         normal_idx++;
11538                         mdest_res->sample_idx[dest_index].rix_jump =
11539                                 dev_flow->handle->rix_jump;
11540                         sample_act->dr_jump_action =
11541                                 dev_flow->dv.jump->action;
11542                         dev_flow->handle->rix_jump = 0;
11543                 }
11544                 sample_act->actions_num = normal_idx;
11545                 /* update sample action resource into first index of array */
11546                 mdest_res->ft_type = res->ft_type;
11547                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
11548                                 sizeof(struct mlx5_flow_sub_actions_idx));
11549                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
11550                                 sizeof(struct mlx5_flow_sub_actions_list));
11551                 mdest_res->num_of_dest = num_of_dest;
11552                 if (flow_dv_dest_array_resource_register(dev, mdest_res,
11553                                                          dev_flow, error))
11554                         return rte_flow_error_set(error, EINVAL,
11555                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11556                                                   NULL, "can't create sample "
11557                                                   "action");
11558         } else {
11559                 res->sub_actions = sample_actions;
11560                 if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
11561                         return rte_flow_error_set(error, EINVAL,
11562                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11563                                                   NULL,
11564                                                   "can't create sample action");
11565         }
11566         return 0;
11567 }
11568
11569 /**
11570  * Remove an ASO age action from age actions list.
11571  *
11572  * @param[in] dev
11573  *   Pointer to the Ethernet device structure.
11574  * @param[in] age
11575  *   Pointer to the aso age action handler.
11576  */
11577 static void
11578 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev,
11579                                 struct mlx5_aso_age_action *age)
11580 {
11581         struct mlx5_age_info *age_info;
11582         struct mlx5_age_param *age_param = &age->age_params;
11583         struct mlx5_priv *priv = dev->data->dev_private;
11584         uint16_t expected = AGE_CANDIDATE;
11585
11586         age_info = GET_PORT_AGE_INFO(priv);
11587         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
11588                                          AGE_FREE, false, __ATOMIC_RELAXED,
11589                                          __ATOMIC_RELAXED)) {
11590                 /**
11591                  * We need the lock even it is age timeout,
11592                  * since age action may still in process.
11593                  */
11594                 rte_spinlock_lock(&age_info->aged_sl);
11595                 LIST_REMOVE(age, next);
11596                 rte_spinlock_unlock(&age_info->aged_sl);
11597                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
11598         }
11599 }
11600
11601 /**
11602  * Release an ASO age action.
11603  *
11604  * @param[in] dev
11605  *   Pointer to the Ethernet device structure.
11606  * @param[in] age_idx
11607  *   Index of ASO age action to release.
11608  * @param[in] flow
11609  *   True if the release operation is during flow destroy operation.
11610  *   False if the release operation is during action destroy operation.
11611  *
11612  * @return
11613  *   0 when age action was removed, otherwise the number of references.
11614  */
11615 static int
11616 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
11617 {
11618         struct mlx5_priv *priv = dev->data->dev_private;
11619         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11620         struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx);
11621         uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED);
11622
11623         if (!ret) {
11624                 flow_dv_aso_age_remove_from_age(dev, age);
11625                 rte_spinlock_lock(&mng->free_sl);
11626                 LIST_INSERT_HEAD(&mng->free, age, next);
11627                 rte_spinlock_unlock(&mng->free_sl);
11628         }
11629         return ret;
11630 }
11631
11632 /**
11633  * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
11634  *
11635  * @param[in] dev
11636  *   Pointer to the Ethernet device structure.
11637  *
11638  * @return
11639  *   0 on success, otherwise negative errno value and rte_errno is set.
11640  */
11641 static int
11642 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
11643 {
11644         struct mlx5_priv *priv = dev->data->dev_private;
11645         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11646         void *old_pools = mng->pools;
11647         uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
11648         uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
11649         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
11650
11651         if (!pools) {
11652                 rte_errno = ENOMEM;
11653                 return -ENOMEM;
11654         }
11655         if (old_pools) {
11656                 memcpy(pools, old_pools,
11657                        mng->n * sizeof(struct mlx5_flow_counter_pool *));
11658                 mlx5_free(old_pools);
11659         } else {
11660                 /* First ASO flow hit allocation - starting ASO data-path. */
11661                 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh);
11662
11663                 if (ret) {
11664                         mlx5_free(pools);
11665                         return ret;
11666                 }
11667         }
11668         mng->n = resize;
11669         mng->pools = pools;
11670         return 0;
11671 }
11672
11673 /**
11674  * Create and initialize a new ASO aging pool.
11675  *
11676  * @param[in] dev
11677  *   Pointer to the Ethernet device structure.
11678  * @param[out] age_free
11679  *   Where to put the pointer of a new age action.
11680  *
11681  * @return
11682  *   The age actions pool pointer and @p age_free is set on success,
11683  *   NULL otherwise and rte_errno is set.
11684  */
11685 static struct mlx5_aso_age_pool *
11686 flow_dv_age_pool_create(struct rte_eth_dev *dev,
11687                         struct mlx5_aso_age_action **age_free)
11688 {
11689         struct mlx5_priv *priv = dev->data->dev_private;
11690         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11691         struct mlx5_aso_age_pool *pool = NULL;
11692         struct mlx5_devx_obj *obj = NULL;
11693         uint32_t i;
11694
11695         obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->ctx,
11696                                                     priv->sh->pdn);
11697         if (!obj) {
11698                 rte_errno = ENODATA;
11699                 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX.");
11700                 return NULL;
11701         }
11702         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
11703         if (!pool) {
11704                 claim_zero(mlx5_devx_cmd_destroy(obj));
11705                 rte_errno = ENOMEM;
11706                 return NULL;
11707         }
11708         pool->flow_hit_aso_obj = obj;
11709         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
11710         rte_spinlock_lock(&mng->resize_sl);
11711         pool->index = mng->next;
11712         /* Resize pools array if there is no room for the new pool in it. */
11713         if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) {
11714                 claim_zero(mlx5_devx_cmd_destroy(obj));
11715                 mlx5_free(pool);
11716                 rte_spinlock_unlock(&mng->resize_sl);
11717                 return NULL;
11718         }
11719         mng->pools[pool->index] = pool;
11720         mng->next++;
11721         rte_spinlock_unlock(&mng->resize_sl);
11722         /* Assign the first action in the new pool, the rest go to free list. */
11723         *age_free = &pool->actions[0];
11724         for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) {
11725                 pool->actions[i].offset = i;
11726                 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next);
11727         }
11728         return pool;
11729 }
11730
11731 /**
11732  * Allocate a ASO aging bit.
11733  *
11734  * @param[in] dev
11735  *   Pointer to the Ethernet device structure.
11736  * @param[out] error
11737  *   Pointer to the error structure.
11738  *
11739  * @return
11740  *   Index to ASO age action on success, 0 otherwise and rte_errno is set.
11741  */
11742 static uint32_t
11743 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
11744 {
11745         struct mlx5_priv *priv = dev->data->dev_private;
11746         const struct mlx5_aso_age_pool *pool;
11747         struct mlx5_aso_age_action *age_free = NULL;
11748         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11749
11750         MLX5_ASSERT(mng);
11751         /* Try to get the next free age action bit. */
11752         rte_spinlock_lock(&mng->free_sl);
11753         age_free = LIST_FIRST(&mng->free);
11754         if (age_free) {
11755                 LIST_REMOVE(age_free, next);
11756         } else if (!flow_dv_age_pool_create(dev, &age_free)) {
11757                 rte_spinlock_unlock(&mng->free_sl);
11758                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
11759                                    NULL, "failed to create ASO age pool");
11760                 return 0; /* 0 is an error. */
11761         }
11762         rte_spinlock_unlock(&mng->free_sl);
11763         pool = container_of
11764           ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL])
11765                   (age_free - age_free->offset), const struct mlx5_aso_age_pool,
11766                                                                        actions);
11767         if (!age_free->dr_action) {
11768                 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0,
11769                                                  error);
11770
11771                 if (reg_c < 0) {
11772                         rte_flow_error_set(error, rte_errno,
11773                                            RTE_FLOW_ERROR_TYPE_ACTION,
11774                                            NULL, "failed to get reg_c "
11775                                            "for ASO flow hit");
11776                         return 0; /* 0 is an error. */
11777                 }
11778 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
11779                 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso
11780                                 (priv->sh->rx_domain,
11781                                  pool->flow_hit_aso_obj->obj, age_free->offset,
11782                                  MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET,
11783                                  (reg_c - REG_C_0));
11784 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
11785                 if (!age_free->dr_action) {
11786                         rte_errno = errno;
11787                         rte_spinlock_lock(&mng->free_sl);
11788                         LIST_INSERT_HEAD(&mng->free, age_free, next);
11789                         rte_spinlock_unlock(&mng->free_sl);
11790                         rte_flow_error_set(error, rte_errno,
11791                                            RTE_FLOW_ERROR_TYPE_ACTION,
11792                                            NULL, "failed to create ASO "
11793                                            "flow hit action");
11794                         return 0; /* 0 is an error. */
11795                 }
11796         }
11797         __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED);
11798         return pool->index | ((age_free->offset + 1) << 16);
11799 }
11800
11801 /**
11802  * Initialize flow ASO age parameters.
11803  *
11804  * @param[in] dev
11805  *   Pointer to rte_eth_dev structure.
11806  * @param[in] age_idx
11807  *   Index of ASO age action.
11808  * @param[in] context
11809  *   Pointer to flow counter age context.
11810  * @param[in] timeout
11811  *   Aging timeout in seconds.
11812  *
11813  */
11814 static void
11815 flow_dv_aso_age_params_init(struct rte_eth_dev *dev,
11816                             uint32_t age_idx,
11817                             void *context,
11818                             uint32_t timeout)
11819 {
11820         struct mlx5_aso_age_action *aso_age;
11821
11822         aso_age = flow_aso_age_get_by_idx(dev, age_idx);
11823         MLX5_ASSERT(aso_age);
11824         aso_age->age_params.context = context;
11825         aso_age->age_params.timeout = timeout;
11826         aso_age->age_params.port_id = dev->data->port_id;
11827         __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0,
11828                          __ATOMIC_RELAXED);
11829         __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE,
11830                          __ATOMIC_RELAXED);
11831 }
11832
11833 static void
11834 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask,
11835                                const struct rte_flow_item_integrity *value,
11836                                void *headers_m, void *headers_v)
11837 {
11838         if (mask->l4_ok) {
11839                 /* application l4_ok filter aggregates all hardware l4 filters
11840                  * therefore hw l4_checksum_ok must be implicitly added here.
11841                  */
11842                 struct rte_flow_item_integrity local_item;
11843
11844                 local_item.l4_csum_ok = 1;
11845                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
11846                          local_item.l4_csum_ok);
11847                 if (value->l4_ok) {
11848                         /* application l4_ok = 1 matches sets both hw flags
11849                          * l4_ok and l4_checksum_ok flags to 1.
11850                          */
11851                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11852                                  l4_checksum_ok, local_item.l4_csum_ok);
11853                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok,
11854                                  mask->l4_ok);
11855                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok,
11856                                  value->l4_ok);
11857                 } else {
11858                         /* application l4_ok = 0 matches on hw flag
11859                          * l4_checksum_ok = 0 only.
11860                          */
11861                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11862                                  l4_checksum_ok, 0);
11863                 }
11864         } else if (mask->l4_csum_ok) {
11865                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
11866                          mask->l4_csum_ok);
11867                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
11868                          value->l4_csum_ok);
11869         }
11870 }
11871
11872 static void
11873 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask,
11874                                const struct rte_flow_item_integrity *value,
11875                                void *headers_m, void *headers_v,
11876                                bool is_ipv4)
11877 {
11878         if (mask->l3_ok) {
11879                 /* application l3_ok filter aggregates all hardware l3 filters
11880                  * therefore hw ipv4_checksum_ok must be implicitly added here.
11881                  */
11882                 struct rte_flow_item_integrity local_item;
11883
11884                 local_item.ipv4_csum_ok = !!is_ipv4;
11885                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
11886                          local_item.ipv4_csum_ok);
11887                 if (value->l3_ok) {
11888                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11889                                  ipv4_checksum_ok, local_item.ipv4_csum_ok);
11890                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok,
11891                                  mask->l3_ok);
11892                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok,
11893                                  value->l3_ok);
11894                 } else {
11895                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
11896                                  ipv4_checksum_ok, 0);
11897                 }
11898         } else if (mask->ipv4_csum_ok) {
11899                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
11900                          mask->ipv4_csum_ok);
11901                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
11902                          value->ipv4_csum_ok);
11903         }
11904 }
11905
11906 static void
11907 flow_dv_translate_item_integrity(void *matcher, void *key,
11908                                  const struct rte_flow_item *head_item,
11909                                  const struct rte_flow_item *integrity_item)
11910 {
11911         const struct rte_flow_item_integrity *mask = integrity_item->mask;
11912         const struct rte_flow_item_integrity *value = integrity_item->spec;
11913         const struct rte_flow_item *tunnel_item, *end_item, *item;
11914         void *headers_m;
11915         void *headers_v;
11916         uint32_t l3_protocol;
11917
11918         if (!value)
11919                 return;
11920         if (!mask)
11921                 mask = &rte_flow_item_integrity_mask;
11922         if (value->level > 1) {
11923                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
11924                                          inner_headers);
11925                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
11926         } else {
11927                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
11928                                          outer_headers);
11929                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
11930         }
11931         tunnel_item = mlx5_flow_find_tunnel_item(head_item);
11932         if (value->level > 1) {
11933                 /* tunnel item was verified during the item validation */
11934                 item = tunnel_item;
11935                 end_item = mlx5_find_end_item(tunnel_item);
11936         } else {
11937                 item = head_item;
11938                 end_item = tunnel_item ? tunnel_item :
11939                            mlx5_find_end_item(integrity_item);
11940         }
11941         l3_protocol = mask->l3_ok ?
11942                       mlx5_flow_locate_proto_l3(&item, end_item) : 0;
11943         flow_dv_translate_integrity_l3(mask, value, headers_m, headers_v,
11944                                        l3_protocol == RTE_ETHER_TYPE_IPV4);
11945         flow_dv_translate_integrity_l4(mask, value, headers_m, headers_v);
11946 }
11947
11948 /**
11949  * Prepares DV flow counter with aging configuration.
11950  * Gets it by index when exists, creates a new one when doesn't.
11951  *
11952  * @param[in] dev
11953  *   Pointer to rte_eth_dev structure.
11954  * @param[in] dev_flow
11955  *   Pointer to the mlx5_flow.
11956  * @param[in, out] flow
11957  *   Pointer to the sub flow.
11958  * @param[in] count
11959  *   Pointer to the counter action configuration.
11960  * @param[in] age
11961  *   Pointer to the aging action configuration.
11962  * @param[out] error
11963  *   Pointer to the error structure.
11964  *
11965  * @return
11966  *   Pointer to the counter, NULL otherwise.
11967  */
11968 static struct mlx5_flow_counter *
11969 flow_dv_prepare_counter(struct rte_eth_dev *dev,
11970                         struct mlx5_flow *dev_flow,
11971                         struct rte_flow *flow,
11972                         const struct rte_flow_action_count *count,
11973                         const struct rte_flow_action_age *age,
11974                         struct rte_flow_error *error)
11975 {
11976         if (!flow->counter) {
11977                 flow->counter = flow_dv_translate_create_counter(dev, dev_flow,
11978                                                                  count, age);
11979                 if (!flow->counter) {
11980                         rte_flow_error_set(error, rte_errno,
11981                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11982                                            "cannot create counter object.");
11983                         return NULL;
11984                 }
11985         }
11986         return flow_dv_counter_get_by_idx(dev, flow->counter, NULL);
11987 }
11988
11989 /*
11990  * Release an ASO CT action by its own device.
11991  *
11992  * @param[in] dev
11993  *   Pointer to the Ethernet device structure.
11994  * @param[in] idx
11995  *   Index of ASO CT action to release.
11996  *
11997  * @return
11998  *   0 when CT action was removed, otherwise the number of references.
11999  */
12000 static inline int
12001 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx)
12002 {
12003         struct mlx5_priv *priv = dev->data->dev_private;
12004         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12005         uint32_t ret;
12006         struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12007         enum mlx5_aso_ct_state state =
12008                         __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
12009
12010         /* Cannot release when CT is in the ASO SQ. */
12011         if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY)
12012                 return -1;
12013         ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED);
12014         if (!ret) {
12015                 if (ct->dr_action_orig) {
12016 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12017                         claim_zero(mlx5_glue->destroy_flow_action
12018                                         (ct->dr_action_orig));
12019 #endif
12020                         ct->dr_action_orig = NULL;
12021                 }
12022                 if (ct->dr_action_rply) {
12023 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12024                         claim_zero(mlx5_glue->destroy_flow_action
12025                                         (ct->dr_action_rply));
12026 #endif
12027                         ct->dr_action_rply = NULL;
12028                 }
12029                 /* Clear the state to free, no need in 1st allocation. */
12030                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE);
12031                 rte_spinlock_lock(&mng->ct_sl);
12032                 LIST_INSERT_HEAD(&mng->free_cts, ct, next);
12033                 rte_spinlock_unlock(&mng->ct_sl);
12034         }
12035         return (int)ret;
12036 }
12037
12038 static inline int
12039 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx)
12040 {
12041         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx);
12042         uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx);
12043         struct rte_eth_dev *owndev = &rte_eth_devices[owner];
12044         RTE_SET_USED(dev);
12045
12046         MLX5_ASSERT(owner < RTE_MAX_ETHPORTS);
12047         if (dev->data->dev_started != 1)
12048                 return -1;
12049         return flow_dv_aso_ct_dev_release(owndev, idx);
12050 }
12051
12052 /*
12053  * Resize the ASO CT pools array by 64 pools.
12054  *
12055  * @param[in] dev
12056  *   Pointer to the Ethernet device structure.
12057  *
12058  * @return
12059  *   0 on success, otherwise negative errno value and rte_errno is set.
12060  */
12061 static int
12062 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev)
12063 {
12064         struct mlx5_priv *priv = dev->data->dev_private;
12065         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12066         void *old_pools = mng->pools;
12067         /* Magic number now, need a macro. */
12068         uint32_t resize = mng->n + 64;
12069         uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize;
12070         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
12071
12072         if (!pools) {
12073                 rte_errno = ENOMEM;
12074                 return -rte_errno;
12075         }
12076         rte_rwlock_write_lock(&mng->resize_rwl);
12077         /* ASO SQ/QP was already initialized in the startup. */
12078         if (old_pools) {
12079                 /* Realloc could be an alternative choice. */
12080                 rte_memcpy(pools, old_pools,
12081                            mng->n * sizeof(struct mlx5_aso_ct_pool *));
12082                 mlx5_free(old_pools);
12083         }
12084         mng->n = resize;
12085         mng->pools = pools;
12086         rte_rwlock_write_unlock(&mng->resize_rwl);
12087         return 0;
12088 }
12089
12090 /*
12091  * Create and initialize a new ASO CT pool.
12092  *
12093  * @param[in] dev
12094  *   Pointer to the Ethernet device structure.
12095  * @param[out] ct_free
12096  *   Where to put the pointer of a new CT action.
12097  *
12098  * @return
12099  *   The CT actions pool pointer and @p ct_free is set on success,
12100  *   NULL otherwise and rte_errno is set.
12101  */
12102 static struct mlx5_aso_ct_pool *
12103 flow_dv_ct_pool_create(struct rte_eth_dev *dev,
12104                        struct mlx5_aso_ct_action **ct_free)
12105 {
12106         struct mlx5_priv *priv = dev->data->dev_private;
12107         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12108         struct mlx5_aso_ct_pool *pool = NULL;
12109         struct mlx5_devx_obj *obj = NULL;
12110         uint32_t i;
12111         uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL);
12112
12113         obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->ctx,
12114                                                 priv->sh->pdn, log_obj_size);
12115         if (!obj) {
12116                 rte_errno = ENODATA;
12117                 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
12118                 return NULL;
12119         }
12120         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
12121         if (!pool) {
12122                 rte_errno = ENOMEM;
12123                 claim_zero(mlx5_devx_cmd_destroy(obj));
12124                 return NULL;
12125         }
12126         pool->devx_obj = obj;
12127         pool->index = mng->next;
12128         /* Resize pools array if there is no room for the new pool in it. */
12129         if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) {
12130                 claim_zero(mlx5_devx_cmd_destroy(obj));
12131                 mlx5_free(pool);
12132                 return NULL;
12133         }
12134         mng->pools[pool->index] = pool;
12135         mng->next++;
12136         /* Assign the first action in the new pool, the rest go to free list. */
12137         *ct_free = &pool->actions[0];
12138         /* Lock outside, the list operation is safe here. */
12139         for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) {
12140                 /* refcnt is 0 when allocating the memory. */
12141                 pool->actions[i].offset = i;
12142                 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next);
12143         }
12144         return pool;
12145 }
12146
12147 /*
12148  * Allocate a ASO CT action from free list.
12149  *
12150  * @param[in] dev
12151  *   Pointer to the Ethernet device structure.
12152  * @param[out] error
12153  *   Pointer to the error structure.
12154  *
12155  * @return
12156  *   Index to ASO CT action on success, 0 otherwise and rte_errno is set.
12157  */
12158 static uint32_t
12159 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12160 {
12161         struct mlx5_priv *priv = dev->data->dev_private;
12162         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12163         struct mlx5_aso_ct_action *ct = NULL;
12164         struct mlx5_aso_ct_pool *pool;
12165         uint8_t reg_c;
12166         uint32_t ct_idx;
12167
12168         MLX5_ASSERT(mng);
12169         if (!priv->config.devx) {
12170                 rte_errno = ENOTSUP;
12171                 return 0;
12172         }
12173         /* Get a free CT action, if no, a new pool will be created. */
12174         rte_spinlock_lock(&mng->ct_sl);
12175         ct = LIST_FIRST(&mng->free_cts);
12176         if (ct) {
12177                 LIST_REMOVE(ct, next);
12178         } else if (!flow_dv_ct_pool_create(dev, &ct)) {
12179                 rte_spinlock_unlock(&mng->ct_sl);
12180                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12181                                    NULL, "failed to create ASO CT pool");
12182                 return 0;
12183         }
12184         rte_spinlock_unlock(&mng->ct_sl);
12185         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
12186         ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset);
12187         /* 0: inactive, 1: created, 2+: used by flows. */
12188         __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED);
12189         reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error);
12190         if (!ct->dr_action_orig) {
12191 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12192                 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso
12193                         (priv->sh->rx_domain, pool->devx_obj->obj,
12194                          ct->offset,
12195                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR,
12196                          reg_c - REG_C_0);
12197 #else
12198                 RTE_SET_USED(reg_c);
12199 #endif
12200                 if (!ct->dr_action_orig) {
12201                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12202                         rte_flow_error_set(error, rte_errno,
12203                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12204                                            "failed to create ASO CT action");
12205                         return 0;
12206                 }
12207         }
12208         if (!ct->dr_action_rply) {
12209 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12210                 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso
12211                         (priv->sh->rx_domain, pool->devx_obj->obj,
12212                          ct->offset,
12213                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER,
12214                          reg_c - REG_C_0);
12215 #endif
12216                 if (!ct->dr_action_rply) {
12217                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12218                         rte_flow_error_set(error, rte_errno,
12219                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12220                                            "failed to create ASO CT action");
12221                         return 0;
12222                 }
12223         }
12224         return ct_idx;
12225 }
12226
12227 /*
12228  * Create a conntrack object with context and actions by using ASO mechanism.
12229  *
12230  * @param[in] dev
12231  *   Pointer to rte_eth_dev structure.
12232  * @param[in] pro
12233  *   Pointer to conntrack information profile.
12234  * @param[out] error
12235  *   Pointer to the error structure.
12236  *
12237  * @return
12238  *   Index to conntrack object on success, 0 otherwise.
12239  */
12240 static uint32_t
12241 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev,
12242                                    const struct rte_flow_action_conntrack *pro,
12243                                    struct rte_flow_error *error)
12244 {
12245         struct mlx5_priv *priv = dev->data->dev_private;
12246         struct mlx5_dev_ctx_shared *sh = priv->sh;
12247         struct mlx5_aso_ct_action *ct;
12248         uint32_t idx;
12249
12250         if (!sh->ct_aso_en)
12251                 return rte_flow_error_set(error, ENOTSUP,
12252                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12253                                           "Connection is not supported");
12254         idx = flow_dv_aso_ct_alloc(dev, error);
12255         if (!idx)
12256                 return rte_flow_error_set(error, rte_errno,
12257                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12258                                           "Failed to allocate CT object");
12259         ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12260         if (mlx5_aso_ct_update_by_wqe(sh, ct, pro))
12261                 return rte_flow_error_set(error, EBUSY,
12262                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12263                                           "Failed to update CT");
12264         ct->is_original = !!pro->is_original_dir;
12265         ct->peer = pro->peer_port;
12266         return idx;
12267 }
12268
12269 /**
12270  * Fill the flow with DV spec, lock free
12271  * (mutex should be acquired by caller).
12272  *
12273  * @param[in] dev
12274  *   Pointer to rte_eth_dev structure.
12275  * @param[in, out] dev_flow
12276  *   Pointer to the sub flow.
12277  * @param[in] attr
12278  *   Pointer to the flow attributes.
12279  * @param[in] items
12280  *   Pointer to the list of items.
12281  * @param[in] actions
12282  *   Pointer to the list of actions.
12283  * @param[out] error
12284  *   Pointer to the error structure.
12285  *
12286  * @return
12287  *   0 on success, a negative errno value otherwise and rte_errno is set.
12288  */
12289 static int
12290 flow_dv_translate(struct rte_eth_dev *dev,
12291                   struct mlx5_flow *dev_flow,
12292                   const struct rte_flow_attr *attr,
12293                   const struct rte_flow_item items[],
12294                   const struct rte_flow_action actions[],
12295                   struct rte_flow_error *error)
12296 {
12297         struct mlx5_priv *priv = dev->data->dev_private;
12298         struct mlx5_dev_config *dev_conf = &priv->config;
12299         struct rte_flow *flow = dev_flow->flow;
12300         struct mlx5_flow_handle *handle = dev_flow->handle;
12301         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12302         struct mlx5_flow_rss_desc *rss_desc;
12303         uint64_t item_flags = 0;
12304         uint64_t last_item = 0;
12305         uint64_t action_flags = 0;
12306         struct mlx5_flow_dv_matcher matcher = {
12307                 .mask = {
12308                         .size = sizeof(matcher.mask.buf),
12309                 },
12310         };
12311         int actions_n = 0;
12312         bool actions_end = false;
12313         union {
12314                 struct mlx5_flow_dv_modify_hdr_resource res;
12315                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
12316                             sizeof(struct mlx5_modification_cmd) *
12317                             (MLX5_MAX_MODIFY_NUM + 1)];
12318         } mhdr_dummy;
12319         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
12320         const struct rte_flow_action_count *count = NULL;
12321         const struct rte_flow_action_age *non_shared_age = NULL;
12322         union flow_dv_attr flow_attr = { .attr = 0 };
12323         uint32_t tag_be;
12324         union mlx5_flow_tbl_key tbl_key;
12325         uint32_t modify_action_position = UINT32_MAX;
12326         void *match_mask = matcher.mask.buf;
12327         void *match_value = dev_flow->dv.value.buf;
12328         uint8_t next_protocol = 0xff;
12329         struct rte_vlan_hdr vlan = { 0 };
12330         struct mlx5_flow_dv_dest_array_resource mdest_res;
12331         struct mlx5_flow_dv_sample_resource sample_res;
12332         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
12333         const struct rte_flow_action_sample *sample = NULL;
12334         struct mlx5_flow_sub_actions_list *sample_act;
12335         uint32_t sample_act_pos = UINT32_MAX;
12336         uint32_t age_act_pos = UINT32_MAX;
12337         uint32_t num_of_dest = 0;
12338         int tmp_actions_n = 0;
12339         uint32_t table;
12340         int ret = 0;
12341         const struct mlx5_flow_tunnel *tunnel = NULL;
12342         struct flow_grp_info grp_info = {
12343                 .external = !!dev_flow->external,
12344                 .transfer = !!attr->transfer,
12345                 .fdb_def_rule = !!priv->fdb_def_rule,
12346                 .skip_scale = dev_flow->skip_scale &
12347                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
12348                 .std_tbl_fix = true,
12349         };
12350         const struct rte_flow_item *head_item = items;
12351
12352         if (!wks)
12353                 return rte_flow_error_set(error, ENOMEM,
12354                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12355                                           NULL,
12356                                           "failed to push flow workspace");
12357         rss_desc = &wks->rss_desc;
12358         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
12359         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
12360         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12361                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12362         /* update normal path action resource into last index of array */
12363         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
12364         if (is_tunnel_offload_active(dev)) {
12365                 if (dev_flow->tunnel) {
12366                         RTE_VERIFY(dev_flow->tof_type ==
12367                                    MLX5_TUNNEL_OFFLOAD_MISS_RULE);
12368                         tunnel = dev_flow->tunnel;
12369                 } else {
12370                         tunnel = mlx5_get_tof(items, actions,
12371                                               &dev_flow->tof_type);
12372                         dev_flow->tunnel = tunnel;
12373                 }
12374                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
12375                                         (dev, attr, tunnel, dev_flow->tof_type);
12376         }
12377         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12378                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12379         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
12380                                        &grp_info, error);
12381         if (ret)
12382                 return ret;
12383         dev_flow->dv.group = table;
12384         if (attr->transfer)
12385                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12386         /* number of actions must be set to 0 in case of dirty stack. */
12387         mhdr_res->actions_num = 0;
12388         if (is_flow_tunnel_match_rule(dev_flow->tof_type)) {
12389                 /*
12390                  * do not add decap action if match rule drops packet
12391                  * HW rejects rules with decap & drop
12392                  *
12393                  * if tunnel match rule was inserted before matching tunnel set
12394                  * rule flow table used in the match rule must be registered.
12395                  * current implementation handles that in the
12396                  * flow_dv_match_register() at the function end.
12397                  */
12398                 bool add_decap = true;
12399                 const struct rte_flow_action *ptr = actions;
12400
12401                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
12402                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
12403                                 add_decap = false;
12404                                 break;
12405                         }
12406                 }
12407                 if (add_decap) {
12408                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12409                                                            attr->transfer,
12410                                                            error))
12411                                 return -rte_errno;
12412                         dev_flow->dv.actions[actions_n++] =
12413                                         dev_flow->dv.encap_decap->action;
12414                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12415                 }
12416         }
12417         for (; !actions_end ; actions++) {
12418                 const struct rte_flow_action_queue *queue;
12419                 const struct rte_flow_action_rss *rss;
12420                 const struct rte_flow_action *action = actions;
12421                 const uint8_t *rss_key;
12422                 struct mlx5_flow_tbl_resource *tbl;
12423                 struct mlx5_aso_age_action *age_act;
12424                 struct mlx5_flow_counter *cnt_act;
12425                 uint32_t port_id = 0;
12426                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
12427                 int action_type = actions->type;
12428                 const struct rte_flow_action *found_action = NULL;
12429                 uint32_t jump_group = 0;
12430                 uint32_t owner_idx;
12431                 struct mlx5_aso_ct_action *ct;
12432
12433                 if (!mlx5_flow_os_action_supported(action_type))
12434                         return rte_flow_error_set(error, ENOTSUP,
12435                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12436                                                   actions,
12437                                                   "action not supported");
12438                 switch (action_type) {
12439                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
12440                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
12441                         break;
12442                 case RTE_FLOW_ACTION_TYPE_VOID:
12443                         break;
12444                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
12445                         if (flow_dv_translate_action_port_id(dev, action,
12446                                                              &port_id, error))
12447                                 return -rte_errno;
12448                         port_id_resource.port_id = port_id;
12449                         MLX5_ASSERT(!handle->rix_port_id_action);
12450                         if (flow_dv_port_id_action_resource_register
12451                             (dev, &port_id_resource, dev_flow, error))
12452                                 return -rte_errno;
12453                         dev_flow->dv.actions[actions_n++] =
12454                                         dev_flow->dv.port_id_action->action;
12455                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12456                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
12457                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12458                         num_of_dest++;
12459                         break;
12460                 case RTE_FLOW_ACTION_TYPE_FLAG:
12461                         action_flags |= MLX5_FLOW_ACTION_FLAG;
12462                         dev_flow->handle->mark = 1;
12463                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12464                                 struct rte_flow_action_mark mark = {
12465                                         .id = MLX5_FLOW_MARK_DEFAULT,
12466                                 };
12467
12468                                 if (flow_dv_convert_action_mark(dev, &mark,
12469                                                                 mhdr_res,
12470                                                                 error))
12471                                         return -rte_errno;
12472                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12473                                 break;
12474                         }
12475                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
12476                         /*
12477                          * Only one FLAG or MARK is supported per device flow
12478                          * right now. So the pointer to the tag resource must be
12479                          * zero before the register process.
12480                          */
12481                         MLX5_ASSERT(!handle->dvh.rix_tag);
12482                         if (flow_dv_tag_resource_register(dev, tag_be,
12483                                                           dev_flow, error))
12484                                 return -rte_errno;
12485                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12486                         dev_flow->dv.actions[actions_n++] =
12487                                         dev_flow->dv.tag_resource->action;
12488                         break;
12489                 case RTE_FLOW_ACTION_TYPE_MARK:
12490                         action_flags |= MLX5_FLOW_ACTION_MARK;
12491                         dev_flow->handle->mark = 1;
12492                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12493                                 const struct rte_flow_action_mark *mark =
12494                                         (const struct rte_flow_action_mark *)
12495                                                 actions->conf;
12496
12497                                 if (flow_dv_convert_action_mark(dev, mark,
12498                                                                 mhdr_res,
12499                                                                 error))
12500                                         return -rte_errno;
12501                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12502                                 break;
12503                         }
12504                         /* Fall-through */
12505                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
12506                         /* Legacy (non-extensive) MARK action. */
12507                         tag_be = mlx5_flow_mark_set
12508                               (((const struct rte_flow_action_mark *)
12509                                (actions->conf))->id);
12510                         MLX5_ASSERT(!handle->dvh.rix_tag);
12511                         if (flow_dv_tag_resource_register(dev, tag_be,
12512                                                           dev_flow, error))
12513                                 return -rte_errno;
12514                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12515                         dev_flow->dv.actions[actions_n++] =
12516                                         dev_flow->dv.tag_resource->action;
12517                         break;
12518                 case RTE_FLOW_ACTION_TYPE_SET_META:
12519                         if (flow_dv_convert_action_set_meta
12520                                 (dev, mhdr_res, attr,
12521                                  (const struct rte_flow_action_set_meta *)
12522                                   actions->conf, error))
12523                                 return -rte_errno;
12524                         action_flags |= MLX5_FLOW_ACTION_SET_META;
12525                         break;
12526                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
12527                         if (flow_dv_convert_action_set_tag
12528                                 (dev, mhdr_res,
12529                                  (const struct rte_flow_action_set_tag *)
12530                                   actions->conf, error))
12531                                 return -rte_errno;
12532                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12533                         break;
12534                 case RTE_FLOW_ACTION_TYPE_DROP:
12535                         action_flags |= MLX5_FLOW_ACTION_DROP;
12536                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
12537                         break;
12538                 case RTE_FLOW_ACTION_TYPE_QUEUE:
12539                         queue = actions->conf;
12540                         rss_desc->queue_num = 1;
12541                         rss_desc->queue[0] = queue->index;
12542                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
12543                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
12544                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
12545                         num_of_dest++;
12546                         break;
12547                 case RTE_FLOW_ACTION_TYPE_RSS:
12548                         rss = actions->conf;
12549                         memcpy(rss_desc->queue, rss->queue,
12550                                rss->queue_num * sizeof(uint16_t));
12551                         rss_desc->queue_num = rss->queue_num;
12552                         /* NULL RSS key indicates default RSS key. */
12553                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
12554                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
12555                         /*
12556                          * rss->level and rss.types should be set in advance
12557                          * when expanding items for RSS.
12558                          */
12559                         action_flags |= MLX5_FLOW_ACTION_RSS;
12560                         dev_flow->handle->fate_action = rss_desc->shared_rss ?
12561                                 MLX5_FLOW_FATE_SHARED_RSS :
12562                                 MLX5_FLOW_FATE_QUEUE;
12563                         break;
12564                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
12565                         flow->age = (uint32_t)(uintptr_t)(action->conf);
12566                         age_act = flow_aso_age_get_by_idx(dev, flow->age);
12567                         __atomic_fetch_add(&age_act->refcnt, 1,
12568                                            __ATOMIC_RELAXED);
12569                         age_act_pos = actions_n++;
12570                         action_flags |= MLX5_FLOW_ACTION_AGE;
12571                         break;
12572                 case RTE_FLOW_ACTION_TYPE_AGE:
12573                         non_shared_age = action->conf;
12574                         age_act_pos = actions_n++;
12575                         action_flags |= MLX5_FLOW_ACTION_AGE;
12576                         break;
12577                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
12578                         flow->counter = (uint32_t)(uintptr_t)(action->conf);
12579                         cnt_act = flow_dv_counter_get_by_idx(dev, flow->counter,
12580                                                              NULL);
12581                         __atomic_fetch_add(&cnt_act->shared_info.refcnt, 1,
12582                                            __ATOMIC_RELAXED);
12583                         /* Save information first, will apply later. */
12584                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12585                         break;
12586                 case RTE_FLOW_ACTION_TYPE_COUNT:
12587                         if (!dev_conf->devx) {
12588                                 return rte_flow_error_set
12589                                               (error, ENOTSUP,
12590                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12591                                                NULL,
12592                                                "count action not supported");
12593                         }
12594                         /* Save information first, will apply later. */
12595                         count = action->conf;
12596                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12597                         break;
12598                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
12599                         dev_flow->dv.actions[actions_n++] =
12600                                                 priv->sh->pop_vlan_action;
12601                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
12602                         break;
12603                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
12604                         if (!(action_flags &
12605                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
12606                                 flow_dev_get_vlan_info_from_items(items, &vlan);
12607                         vlan.eth_proto = rte_be_to_cpu_16
12608                              ((((const struct rte_flow_action_of_push_vlan *)
12609                                                    actions->conf)->ethertype));
12610                         found_action = mlx5_flow_find_action
12611                                         (actions + 1,
12612                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
12613                         if (found_action)
12614                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12615                         found_action = mlx5_flow_find_action
12616                                         (actions + 1,
12617                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
12618                         if (found_action)
12619                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12620                         if (flow_dv_create_action_push_vlan
12621                                             (dev, attr, &vlan, dev_flow, error))
12622                                 return -rte_errno;
12623                         dev_flow->dv.actions[actions_n++] =
12624                                         dev_flow->dv.push_vlan_res->action;
12625                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
12626                         break;
12627                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
12628                         /* of_vlan_push action handled this action */
12629                         MLX5_ASSERT(action_flags &
12630                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
12631                         break;
12632                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
12633                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
12634                                 break;
12635                         flow_dev_get_vlan_info_from_items(items, &vlan);
12636                         mlx5_update_vlan_vid_pcp(actions, &vlan);
12637                         /* If no VLAN push - this is a modify header action */
12638                         if (flow_dv_convert_action_modify_vlan_vid
12639                                                 (mhdr_res, actions, error))
12640                                 return -rte_errno;
12641                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
12642                         break;
12643                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
12644                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
12645                         if (flow_dv_create_action_l2_encap(dev, actions,
12646                                                            dev_flow,
12647                                                            attr->transfer,
12648                                                            error))
12649                                 return -rte_errno;
12650                         dev_flow->dv.actions[actions_n++] =
12651                                         dev_flow->dv.encap_decap->action;
12652                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12653                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12654                                 sample_act->action_flags |=
12655                                                         MLX5_FLOW_ACTION_ENCAP;
12656                         break;
12657                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
12658                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
12659                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12660                                                            attr->transfer,
12661                                                            error))
12662                                 return -rte_errno;
12663                         dev_flow->dv.actions[actions_n++] =
12664                                         dev_flow->dv.encap_decap->action;
12665                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12666                         break;
12667                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
12668                         /* Handle encap with preceding decap. */
12669                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
12670                                 if (flow_dv_create_action_raw_encap
12671                                         (dev, actions, dev_flow, attr, error))
12672                                         return -rte_errno;
12673                                 dev_flow->dv.actions[actions_n++] =
12674                                         dev_flow->dv.encap_decap->action;
12675                         } else {
12676                                 /* Handle encap without preceding decap. */
12677                                 if (flow_dv_create_action_l2_encap
12678                                     (dev, actions, dev_flow, attr->transfer,
12679                                      error))
12680                                         return -rte_errno;
12681                                 dev_flow->dv.actions[actions_n++] =
12682                                         dev_flow->dv.encap_decap->action;
12683                         }
12684                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12685                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12686                                 sample_act->action_flags |=
12687                                                         MLX5_FLOW_ACTION_ENCAP;
12688                         break;
12689                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
12690                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
12691                                 ;
12692                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
12693                                 if (flow_dv_create_action_l2_decap
12694                                     (dev, dev_flow, attr->transfer, error))
12695                                         return -rte_errno;
12696                                 dev_flow->dv.actions[actions_n++] =
12697                                         dev_flow->dv.encap_decap->action;
12698                         }
12699                         /* If decap is followed by encap, handle it at encap. */
12700                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12701                         break;
12702                 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP:
12703                         dev_flow->dv.actions[actions_n++] =
12704                                 (void *)(uintptr_t)action->conf;
12705                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12706                         break;
12707                 case RTE_FLOW_ACTION_TYPE_JUMP:
12708                         jump_group = ((const struct rte_flow_action_jump *)
12709                                                         action->conf)->group;
12710                         grp_info.std_tbl_fix = 0;
12711                         if (dev_flow->skip_scale &
12712                                 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT))
12713                                 grp_info.skip_scale = 1;
12714                         else
12715                                 grp_info.skip_scale = 0;
12716                         ret = mlx5_flow_group_to_table(dev, tunnel,
12717                                                        jump_group,
12718                                                        &table,
12719                                                        &grp_info, error);
12720                         if (ret)
12721                                 return ret;
12722                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
12723                                                        attr->transfer,
12724                                                        !!dev_flow->external,
12725                                                        tunnel, jump_group, 0,
12726                                                        0, error);
12727                         if (!tbl)
12728                                 return rte_flow_error_set
12729                                                 (error, errno,
12730                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12731                                                  NULL,
12732                                                  "cannot create jump action.");
12733                         if (flow_dv_jump_tbl_resource_register
12734                             (dev, tbl, dev_flow, error)) {
12735                                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
12736                                 return rte_flow_error_set
12737                                                 (error, errno,
12738                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12739                                                  NULL,
12740                                                  "cannot create jump action.");
12741                         }
12742                         dev_flow->dv.actions[actions_n++] =
12743                                         dev_flow->dv.jump->action;
12744                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12745                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
12746                         sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP;
12747                         num_of_dest++;
12748                         break;
12749                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
12750                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
12751                         if (flow_dv_convert_action_modify_mac
12752                                         (mhdr_res, actions, error))
12753                                 return -rte_errno;
12754                         action_flags |= actions->type ==
12755                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
12756                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
12757                                         MLX5_FLOW_ACTION_SET_MAC_DST;
12758                         break;
12759                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
12760                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
12761                         if (flow_dv_convert_action_modify_ipv4
12762                                         (mhdr_res, actions, error))
12763                                 return -rte_errno;
12764                         action_flags |= actions->type ==
12765                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
12766                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
12767                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
12768                         break;
12769                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
12770                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
12771                         if (flow_dv_convert_action_modify_ipv6
12772                                         (mhdr_res, actions, error))
12773                                 return -rte_errno;
12774                         action_flags |= actions->type ==
12775                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
12776                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
12777                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
12778                         break;
12779                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
12780                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
12781                         if (flow_dv_convert_action_modify_tp
12782                                         (mhdr_res, actions, items,
12783                                          &flow_attr, dev_flow, !!(action_flags &
12784                                          MLX5_FLOW_ACTION_DECAP), error))
12785                                 return -rte_errno;
12786                         action_flags |= actions->type ==
12787                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
12788                                         MLX5_FLOW_ACTION_SET_TP_SRC :
12789                                         MLX5_FLOW_ACTION_SET_TP_DST;
12790                         break;
12791                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
12792                         if (flow_dv_convert_action_modify_dec_ttl
12793                                         (mhdr_res, items, &flow_attr, dev_flow,
12794                                          !!(action_flags &
12795                                          MLX5_FLOW_ACTION_DECAP), error))
12796                                 return -rte_errno;
12797                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
12798                         break;
12799                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
12800                         if (flow_dv_convert_action_modify_ttl
12801                                         (mhdr_res, actions, items, &flow_attr,
12802                                          dev_flow, !!(action_flags &
12803                                          MLX5_FLOW_ACTION_DECAP), error))
12804                                 return -rte_errno;
12805                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
12806                         break;
12807                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
12808                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
12809                         if (flow_dv_convert_action_modify_tcp_seq
12810                                         (mhdr_res, actions, error))
12811                                 return -rte_errno;
12812                         action_flags |= actions->type ==
12813                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
12814                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
12815                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
12816                         break;
12817
12818                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
12819                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
12820                         if (flow_dv_convert_action_modify_tcp_ack
12821                                         (mhdr_res, actions, error))
12822                                 return -rte_errno;
12823                         action_flags |= actions->type ==
12824                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
12825                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
12826                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
12827                         break;
12828                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
12829                         if (flow_dv_convert_action_set_reg
12830                                         (mhdr_res, actions, error))
12831                                 return -rte_errno;
12832                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12833                         break;
12834                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
12835                         if (flow_dv_convert_action_copy_mreg
12836                                         (dev, mhdr_res, actions, error))
12837                                 return -rte_errno;
12838                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12839                         break;
12840                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
12841                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
12842                         dev_flow->handle->fate_action =
12843                                         MLX5_FLOW_FATE_DEFAULT_MISS;
12844                         break;
12845                 case RTE_FLOW_ACTION_TYPE_METER:
12846                         if (!wks->fm)
12847                                 return rte_flow_error_set(error, rte_errno,
12848                                         RTE_FLOW_ERROR_TYPE_ACTION,
12849                                         NULL, "Failed to get meter in flow.");
12850                         /* Set the meter action. */
12851                         dev_flow->dv.actions[actions_n++] =
12852                                 wks->fm->meter_action;
12853                         action_flags |= MLX5_FLOW_ACTION_METER;
12854                         break;
12855                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
12856                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
12857                                                               actions, error))
12858                                 return -rte_errno;
12859                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
12860                         break;
12861                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
12862                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
12863                                                               actions, error))
12864                                 return -rte_errno;
12865                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
12866                         break;
12867                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
12868                         sample_act_pos = actions_n;
12869                         sample = (const struct rte_flow_action_sample *)
12870                                  action->conf;
12871                         actions_n++;
12872                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
12873                         /* put encap action into group if work with port id */
12874                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
12875                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
12876                                 sample_act->action_flags |=
12877                                                         MLX5_FLOW_ACTION_ENCAP;
12878                         break;
12879                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
12880                         if (flow_dv_convert_action_modify_field
12881                                         (dev, mhdr_res, actions, attr, error))
12882                                 return -rte_errno;
12883                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
12884                         break;
12885                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
12886                         owner_idx = (uint32_t)(uintptr_t)action->conf;
12887                         ct = flow_aso_ct_get_by_idx(dev, owner_idx);
12888                         if (!ct)
12889                                 return rte_flow_error_set(error, EINVAL,
12890                                                 RTE_FLOW_ERROR_TYPE_ACTION,
12891                                                 NULL,
12892                                                 "Failed to get CT object.");
12893                         if (mlx5_aso_ct_available(priv->sh, ct))
12894                                 return rte_flow_error_set(error, rte_errno,
12895                                                 RTE_FLOW_ERROR_TYPE_ACTION,
12896                                                 NULL,
12897                                                 "CT is unavailable.");
12898                         if (ct->is_original)
12899                                 dev_flow->dv.actions[actions_n] =
12900                                                         ct->dr_action_orig;
12901                         else
12902                                 dev_flow->dv.actions[actions_n] =
12903                                                         ct->dr_action_rply;
12904                         flow->indirect_type = MLX5_INDIRECT_ACTION_TYPE_CT;
12905                         flow->ct = owner_idx;
12906                         __atomic_fetch_add(&ct->refcnt, 1, __ATOMIC_RELAXED);
12907                         actions_n++;
12908                         action_flags |= MLX5_FLOW_ACTION_CT;
12909                         break;
12910                 case RTE_FLOW_ACTION_TYPE_END:
12911                         actions_end = true;
12912                         if (mhdr_res->actions_num) {
12913                                 /* create modify action if needed. */
12914                                 if (flow_dv_modify_hdr_resource_register
12915                                         (dev, mhdr_res, dev_flow, error))
12916                                         return -rte_errno;
12917                                 dev_flow->dv.actions[modify_action_position] =
12918                                         handle->dvh.modify_hdr->action;
12919                         }
12920                         /*
12921                          * Handle AGE and COUNT action by single HW counter
12922                          * when they are not shared.
12923                          */
12924                         if (action_flags & MLX5_FLOW_ACTION_AGE) {
12925                                 if ((non_shared_age &&
12926                                      count && !count->shared) ||
12927                                     !(priv->sh->flow_hit_aso_en &&
12928                                       (attr->group || attr->transfer))) {
12929                                         /* Creates age by counters. */
12930                                         cnt_act = flow_dv_prepare_counter
12931                                                                 (dev, dev_flow,
12932                                                                  flow, count,
12933                                                                  non_shared_age,
12934                                                                  error);
12935                                         if (!cnt_act)
12936                                                 return -rte_errno;
12937                                         dev_flow->dv.actions[age_act_pos] =
12938                                                                 cnt_act->action;
12939                                         break;
12940                                 }
12941                                 if (!flow->age && non_shared_age) {
12942                                         flow->age = flow_dv_aso_age_alloc
12943                                                                 (dev, error);
12944                                         if (!flow->age)
12945                                                 return -rte_errno;
12946                                         flow_dv_aso_age_params_init
12947                                                     (dev, flow->age,
12948                                                      non_shared_age->context ?
12949                                                      non_shared_age->context :
12950                                                      (void *)(uintptr_t)
12951                                                      (dev_flow->flow_idx),
12952                                                      non_shared_age->timeout);
12953                                 }
12954                                 age_act = flow_aso_age_get_by_idx(dev,
12955                                                                   flow->age);
12956                                 dev_flow->dv.actions[age_act_pos] =
12957                                                              age_act->dr_action;
12958                         }
12959                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
12960                                 /*
12961                                  * Create one count action, to be used
12962                                  * by all sub-flows.
12963                                  */
12964                                 cnt_act = flow_dv_prepare_counter(dev, dev_flow,
12965                                                                   flow, count,
12966                                                                   NULL, error);
12967                                 if (!cnt_act)
12968                                         return -rte_errno;
12969                                 dev_flow->dv.actions[actions_n++] =
12970                                                                 cnt_act->action;
12971                         }
12972                 default:
12973                         break;
12974                 }
12975                 if (mhdr_res->actions_num &&
12976                     modify_action_position == UINT32_MAX)
12977                         modify_action_position = actions_n++;
12978         }
12979         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
12980                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
12981                 int item_type = items->type;
12982
12983                 if (!mlx5_flow_os_item_supported(item_type))
12984                         return rte_flow_error_set(error, ENOTSUP,
12985                                                   RTE_FLOW_ERROR_TYPE_ITEM,
12986                                                   NULL, "item not supported");
12987                 switch (item_type) {
12988                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
12989                         flow_dv_translate_item_port_id
12990                                 (dev, match_mask, match_value, items, attr);
12991                         last_item = MLX5_FLOW_ITEM_PORT_ID;
12992                         break;
12993                 case RTE_FLOW_ITEM_TYPE_ETH:
12994                         flow_dv_translate_item_eth(match_mask, match_value,
12995                                                    items, tunnel,
12996                                                    dev_flow->dv.group);
12997                         matcher.priority = action_flags &
12998                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
12999                                         !dev_flow->external ?
13000                                         MLX5_PRIORITY_MAP_L3 :
13001                                         MLX5_PRIORITY_MAP_L2;
13002                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
13003                                              MLX5_FLOW_LAYER_OUTER_L2;
13004                         break;
13005                 case RTE_FLOW_ITEM_TYPE_VLAN:
13006                         flow_dv_translate_item_vlan(dev_flow,
13007                                                     match_mask, match_value,
13008                                                     items, tunnel,
13009                                                     dev_flow->dv.group);
13010                         matcher.priority = MLX5_PRIORITY_MAP_L2;
13011                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
13012                                               MLX5_FLOW_LAYER_INNER_VLAN) :
13013                                              (MLX5_FLOW_LAYER_OUTER_L2 |
13014                                               MLX5_FLOW_LAYER_OUTER_VLAN);
13015                         break;
13016                 case RTE_FLOW_ITEM_TYPE_IPV4:
13017                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13018                                                   &item_flags, &tunnel);
13019                         flow_dv_translate_item_ipv4(match_mask, match_value,
13020                                                     items, tunnel,
13021                                                     dev_flow->dv.group);
13022                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13023                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
13024                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
13025                         if (items->mask != NULL &&
13026                             ((const struct rte_flow_item_ipv4 *)
13027                              items->mask)->hdr.next_proto_id) {
13028                                 next_protocol =
13029                                         ((const struct rte_flow_item_ipv4 *)
13030                                          (items->spec))->hdr.next_proto_id;
13031                                 next_protocol &=
13032                                         ((const struct rte_flow_item_ipv4 *)
13033                                          (items->mask))->hdr.next_proto_id;
13034                         } else {
13035                                 /* Reset for inner layer. */
13036                                 next_protocol = 0xff;
13037                         }
13038                         break;
13039                 case RTE_FLOW_ITEM_TYPE_IPV6:
13040                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13041                                                   &item_flags, &tunnel);
13042                         flow_dv_translate_item_ipv6(match_mask, match_value,
13043                                                     items, tunnel,
13044                                                     dev_flow->dv.group);
13045                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13046                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
13047                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
13048                         if (items->mask != NULL &&
13049                             ((const struct rte_flow_item_ipv6 *)
13050                              items->mask)->hdr.proto) {
13051                                 next_protocol =
13052                                         ((const struct rte_flow_item_ipv6 *)
13053                                          items->spec)->hdr.proto;
13054                                 next_protocol &=
13055                                         ((const struct rte_flow_item_ipv6 *)
13056                                          items->mask)->hdr.proto;
13057                         } else {
13058                                 /* Reset for inner layer. */
13059                                 next_protocol = 0xff;
13060                         }
13061                         break;
13062                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
13063                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
13064                                                              match_value,
13065                                                              items, tunnel);
13066                         last_item = tunnel ?
13067                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
13068                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
13069                         if (items->mask != NULL &&
13070                             ((const struct rte_flow_item_ipv6_frag_ext *)
13071                              items->mask)->hdr.next_header) {
13072                                 next_protocol =
13073                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13074                                  items->spec)->hdr.next_header;
13075                                 next_protocol &=
13076                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13077                                  items->mask)->hdr.next_header;
13078                         } else {
13079                                 /* Reset for inner layer. */
13080                                 next_protocol = 0xff;
13081                         }
13082                         break;
13083                 case RTE_FLOW_ITEM_TYPE_TCP:
13084                         flow_dv_translate_item_tcp(match_mask, match_value,
13085                                                    items, tunnel);
13086                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13087                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
13088                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
13089                         break;
13090                 case RTE_FLOW_ITEM_TYPE_UDP:
13091                         flow_dv_translate_item_udp(match_mask, match_value,
13092                                                    items, tunnel);
13093                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13094                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
13095                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
13096                         break;
13097                 case RTE_FLOW_ITEM_TYPE_GRE:
13098                         flow_dv_translate_item_gre(match_mask, match_value,
13099                                                    items, tunnel);
13100                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13101                         last_item = MLX5_FLOW_LAYER_GRE;
13102                         break;
13103                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
13104                         flow_dv_translate_item_gre_key(match_mask,
13105                                                        match_value, items);
13106                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
13107                         break;
13108                 case RTE_FLOW_ITEM_TYPE_NVGRE:
13109                         flow_dv_translate_item_nvgre(match_mask, match_value,
13110                                                      items, tunnel);
13111                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13112                         last_item = MLX5_FLOW_LAYER_GRE;
13113                         break;
13114                 case RTE_FLOW_ITEM_TYPE_VXLAN:
13115                         flow_dv_translate_item_vxlan(dev, attr,
13116                                                      match_mask, match_value,
13117                                                      items, tunnel);
13118                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13119                         last_item = MLX5_FLOW_LAYER_VXLAN;
13120                         break;
13121                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
13122                         flow_dv_translate_item_vxlan_gpe(match_mask,
13123                                                          match_value, items,
13124                                                          tunnel);
13125                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13126                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
13127                         break;
13128                 case RTE_FLOW_ITEM_TYPE_GENEVE:
13129                         flow_dv_translate_item_geneve(match_mask, match_value,
13130                                                       items, tunnel);
13131                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13132                         last_item = MLX5_FLOW_LAYER_GENEVE;
13133                         break;
13134                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
13135                         ret = flow_dv_translate_item_geneve_opt(dev, match_mask,
13136                                                           match_value,
13137                                                           items, error);
13138                         if (ret)
13139                                 return rte_flow_error_set(error, -ret,
13140                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13141                                         "cannot create GENEVE TLV option");
13142                         flow->geneve_tlv_option = 1;
13143                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
13144                         break;
13145                 case RTE_FLOW_ITEM_TYPE_MPLS:
13146                         flow_dv_translate_item_mpls(match_mask, match_value,
13147                                                     items, last_item, tunnel);
13148                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13149                         last_item = MLX5_FLOW_LAYER_MPLS;
13150                         break;
13151                 case RTE_FLOW_ITEM_TYPE_MARK:
13152                         flow_dv_translate_item_mark(dev, match_mask,
13153                                                     match_value, items);
13154                         last_item = MLX5_FLOW_ITEM_MARK;
13155                         break;
13156                 case RTE_FLOW_ITEM_TYPE_META:
13157                         flow_dv_translate_item_meta(dev, match_mask,
13158                                                     match_value, attr, items);
13159                         last_item = MLX5_FLOW_ITEM_METADATA;
13160                         break;
13161                 case RTE_FLOW_ITEM_TYPE_ICMP:
13162                         flow_dv_translate_item_icmp(match_mask, match_value,
13163                                                     items, tunnel);
13164                         last_item = MLX5_FLOW_LAYER_ICMP;
13165                         break;
13166                 case RTE_FLOW_ITEM_TYPE_ICMP6:
13167                         flow_dv_translate_item_icmp6(match_mask, match_value,
13168                                                       items, tunnel);
13169                         last_item = MLX5_FLOW_LAYER_ICMP6;
13170                         break;
13171                 case RTE_FLOW_ITEM_TYPE_TAG:
13172                         flow_dv_translate_item_tag(dev, match_mask,
13173                                                    match_value, items);
13174                         last_item = MLX5_FLOW_ITEM_TAG;
13175                         break;
13176                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
13177                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
13178                                                         match_value, items);
13179                         last_item = MLX5_FLOW_ITEM_TAG;
13180                         break;
13181                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
13182                         flow_dv_translate_item_tx_queue(dev, match_mask,
13183                                                         match_value,
13184                                                         items);
13185                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
13186                         break;
13187                 case RTE_FLOW_ITEM_TYPE_GTP:
13188                         flow_dv_translate_item_gtp(match_mask, match_value,
13189                                                    items, tunnel);
13190                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13191                         last_item = MLX5_FLOW_LAYER_GTP;
13192                         break;
13193                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
13194                         ret = flow_dv_translate_item_gtp_psc(match_mask,
13195                                                           match_value,
13196                                                           items);
13197                         if (ret)
13198                                 return rte_flow_error_set(error, -ret,
13199                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13200                                         "cannot create GTP PSC item");
13201                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
13202                         break;
13203                 case RTE_FLOW_ITEM_TYPE_ECPRI:
13204                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
13205                                 /* Create it only the first time to be used. */
13206                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
13207                                 if (ret)
13208                                         return rte_flow_error_set
13209                                                 (error, -ret,
13210                                                 RTE_FLOW_ERROR_TYPE_ITEM,
13211                                                 NULL,
13212                                                 "cannot create eCPRI parser");
13213                         }
13214                         flow_dv_translate_item_ecpri(dev, match_mask,
13215                                                      match_value, items);
13216                         /* No other protocol should follow eCPRI layer. */
13217                         last_item = MLX5_FLOW_LAYER_ECPRI;
13218                         break;
13219                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
13220                         flow_dv_translate_item_integrity(match_mask,
13221                                                          match_value,
13222                                                          head_item, items);
13223                         break;
13224                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
13225                         flow_dv_translate_item_aso_ct(dev, match_mask,
13226                                                       match_value, items);
13227                         break;
13228                 default:
13229                         break;
13230                 }
13231                 item_flags |= last_item;
13232         }
13233         /*
13234          * When E-Switch mode is enabled, we have two cases where we need to
13235          * set the source port manually.
13236          * The first one, is in case of Nic steering rule, and the second is
13237          * E-Switch rule where no port_id item was found. In both cases
13238          * the source port is set according the current port in use.
13239          */
13240         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
13241             (priv->representor || priv->master)) {
13242                 if (flow_dv_translate_item_port_id(dev, match_mask,
13243                                                    match_value, NULL, attr))
13244                         return -rte_errno;
13245         }
13246 #ifdef RTE_LIBRTE_MLX5_DEBUG
13247         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
13248                                               dev_flow->dv.value.buf));
13249 #endif
13250         /*
13251          * Layers may be already initialized from prefix flow if this dev_flow
13252          * is the suffix flow.
13253          */
13254         handle->layers |= item_flags;
13255         if (action_flags & MLX5_FLOW_ACTION_RSS)
13256                 flow_dv_hashfields_set(dev_flow, rss_desc);
13257         /* If has RSS action in the sample action, the Sample/Mirror resource
13258          * should be registered after the hash filed be update.
13259          */
13260         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
13261                 ret = flow_dv_translate_action_sample(dev,
13262                                                       sample,
13263                                                       dev_flow, attr,
13264                                                       &num_of_dest,
13265                                                       sample_actions,
13266                                                       &sample_res,
13267                                                       error);
13268                 if (ret < 0)
13269                         return ret;
13270                 ret = flow_dv_create_action_sample(dev,
13271                                                    dev_flow,
13272                                                    num_of_dest,
13273                                                    &sample_res,
13274                                                    &mdest_res,
13275                                                    sample_actions,
13276                                                    action_flags,
13277                                                    error);
13278                 if (ret < 0)
13279                         return rte_flow_error_set
13280                                                 (error, rte_errno,
13281                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13282                                                 NULL,
13283                                                 "cannot create sample action");
13284                 if (num_of_dest > 1) {
13285                         dev_flow->dv.actions[sample_act_pos] =
13286                         dev_flow->dv.dest_array_res->action;
13287                 } else {
13288                         dev_flow->dv.actions[sample_act_pos] =
13289                         dev_flow->dv.sample_res->verbs_action;
13290                 }
13291         }
13292         /*
13293          * For multiple destination (sample action with ratio=1), the encap
13294          * action and port id action will be combined into group action.
13295          * So need remove the original these actions in the flow and only
13296          * use the sample action instead of.
13297          */
13298         if (num_of_dest > 1 &&
13299             (sample_act->dr_port_id_action || sample_act->dr_jump_action)) {
13300                 int i;
13301                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
13302
13303                 for (i = 0; i < actions_n; i++) {
13304                         if ((sample_act->dr_encap_action &&
13305                                 sample_act->dr_encap_action ==
13306                                 dev_flow->dv.actions[i]) ||
13307                                 (sample_act->dr_port_id_action &&
13308                                 sample_act->dr_port_id_action ==
13309                                 dev_flow->dv.actions[i]) ||
13310                                 (sample_act->dr_jump_action &&
13311                                 sample_act->dr_jump_action ==
13312                                 dev_flow->dv.actions[i]))
13313                                 continue;
13314                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
13315                 }
13316                 memcpy((void *)dev_flow->dv.actions,
13317                                 (void *)temp_actions,
13318                                 tmp_actions_n * sizeof(void *));
13319                 actions_n = tmp_actions_n;
13320         }
13321         dev_flow->dv.actions_n = actions_n;
13322         dev_flow->act_flags = action_flags;
13323         if (wks->skip_matcher_reg)
13324                 return 0;
13325         /* Register matcher. */
13326         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
13327                                     matcher.mask.size);
13328         matcher.priority = mlx5_get_matcher_priority(dev, attr,
13329                                         matcher.priority);
13330         /**
13331          * When creating meter drop flow in drop table, using original
13332          * 5-tuple match, the matcher priority should be lower than
13333          * mtr_id matcher.
13334          */
13335         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
13336             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP &&
13337             matcher.priority <= MLX5_REG_BITS)
13338                 matcher.priority += MLX5_REG_BITS;
13339         /* reserved field no needs to be set to 0 here. */
13340         tbl_key.is_fdb = attr->transfer;
13341         tbl_key.is_egress = attr->egress;
13342         tbl_key.level = dev_flow->dv.group;
13343         tbl_key.id = dev_flow->dv.table_id;
13344         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
13345                                      tunnel, attr->group, error))
13346                 return -rte_errno;
13347         return 0;
13348 }
13349
13350 /**
13351  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13352  * and tunnel.
13353  *
13354  * @param[in, out] action
13355  *   Shred RSS action holding hash RX queue objects.
13356  * @param[in] hash_fields
13357  *   Defines combination of packet fields to participate in RX hash.
13358  * @param[in] tunnel
13359  *   Tunnel type
13360  * @param[in] hrxq_idx
13361  *   Hash RX queue index to set.
13362  *
13363  * @return
13364  *   0 on success, otherwise negative errno value.
13365  */
13366 static int
13367 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
13368                               const uint64_t hash_fields,
13369                               uint32_t hrxq_idx)
13370 {
13371         uint32_t *hrxqs = action->hrxq;
13372
13373         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13374         case MLX5_RSS_HASH_IPV4:
13375                 /* fall-through. */
13376         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13377                 /* fall-through. */
13378         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13379                 hrxqs[0] = hrxq_idx;
13380                 return 0;
13381         case MLX5_RSS_HASH_IPV4_TCP:
13382                 /* fall-through. */
13383         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13384                 /* fall-through. */
13385         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13386                 hrxqs[1] = hrxq_idx;
13387                 return 0;
13388         case MLX5_RSS_HASH_IPV4_UDP:
13389                 /* fall-through. */
13390         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13391                 /* fall-through. */
13392         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13393                 hrxqs[2] = hrxq_idx;
13394                 return 0;
13395         case MLX5_RSS_HASH_IPV6:
13396                 /* fall-through. */
13397         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13398                 /* fall-through. */
13399         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13400                 hrxqs[3] = hrxq_idx;
13401                 return 0;
13402         case MLX5_RSS_HASH_IPV6_TCP:
13403                 /* fall-through. */
13404         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13405                 /* fall-through. */
13406         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13407                 hrxqs[4] = hrxq_idx;
13408                 return 0;
13409         case MLX5_RSS_HASH_IPV6_UDP:
13410                 /* fall-through. */
13411         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13412                 /* fall-through. */
13413         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13414                 hrxqs[5] = hrxq_idx;
13415                 return 0;
13416         case MLX5_RSS_HASH_NONE:
13417                 hrxqs[6] = hrxq_idx;
13418                 return 0;
13419         default:
13420                 return -1;
13421         }
13422 }
13423
13424 /**
13425  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13426  * and tunnel.
13427  *
13428  * @param[in] dev
13429  *   Pointer to the Ethernet device structure.
13430  * @param[in] idx
13431  *   Shared RSS action ID holding hash RX queue objects.
13432  * @param[in] hash_fields
13433  *   Defines combination of packet fields to participate in RX hash.
13434  * @param[in] tunnel
13435  *   Tunnel type
13436  *
13437  * @return
13438  *   Valid hash RX queue index, otherwise 0.
13439  */
13440 static uint32_t
13441 __flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
13442                                  const uint64_t hash_fields)
13443 {
13444         struct mlx5_priv *priv = dev->data->dev_private;
13445         struct mlx5_shared_action_rss *shared_rss =
13446             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
13447         const uint32_t *hrxqs = shared_rss->hrxq;
13448
13449         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13450         case MLX5_RSS_HASH_IPV4:
13451                 /* fall-through. */
13452         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13453                 /* fall-through. */
13454         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13455                 return hrxqs[0];
13456         case MLX5_RSS_HASH_IPV4_TCP:
13457                 /* fall-through. */
13458         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13459                 /* fall-through. */
13460         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13461                 return hrxqs[1];
13462         case MLX5_RSS_HASH_IPV4_UDP:
13463                 /* fall-through. */
13464         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13465                 /* fall-through. */
13466         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13467                 return hrxqs[2];
13468         case MLX5_RSS_HASH_IPV6:
13469                 /* fall-through. */
13470         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13471                 /* fall-through. */
13472         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13473                 return hrxqs[3];
13474         case MLX5_RSS_HASH_IPV6_TCP:
13475                 /* fall-through. */
13476         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13477                 /* fall-through. */
13478         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13479                 return hrxqs[4];
13480         case MLX5_RSS_HASH_IPV6_UDP:
13481                 /* fall-through. */
13482         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13483                 /* fall-through. */
13484         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13485                 return hrxqs[5];
13486         case MLX5_RSS_HASH_NONE:
13487                 return hrxqs[6];
13488         default:
13489                 return 0;
13490         }
13491
13492 }
13493
13494 /**
13495  * Apply the flow to the NIC, lock free,
13496  * (mutex should be acquired by caller).
13497  *
13498  * @param[in] dev
13499  *   Pointer to the Ethernet device structure.
13500  * @param[in, out] flow
13501  *   Pointer to flow structure.
13502  * @param[out] error
13503  *   Pointer to error structure.
13504  *
13505  * @return
13506  *   0 on success, a negative errno value otherwise and rte_errno is set.
13507  */
13508 static int
13509 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
13510               struct rte_flow_error *error)
13511 {
13512         struct mlx5_flow_dv_workspace *dv;
13513         struct mlx5_flow_handle *dh;
13514         struct mlx5_flow_handle_dv *dv_h;
13515         struct mlx5_flow *dev_flow;
13516         struct mlx5_priv *priv = dev->data->dev_private;
13517         uint32_t handle_idx;
13518         int n;
13519         int err;
13520         int idx;
13521         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
13522         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
13523         uint8_t misc_mask;
13524
13525         MLX5_ASSERT(wks);
13526         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
13527                 dev_flow = &wks->flows[idx];
13528                 dv = &dev_flow->dv;
13529                 dh = dev_flow->handle;
13530                 dv_h = &dh->dvh;
13531                 n = dv->actions_n;
13532                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
13533                         if (dv->transfer) {
13534                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13535                                 dv->actions[n++] = priv->sh->dr_drop_action;
13536                         } else {
13537 #ifdef HAVE_MLX5DV_DR
13538                                 /* DR supports drop action placeholder. */
13539                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13540                                 dv->actions[n++] = priv->sh->dr_drop_action;
13541 #else
13542                                 /* For DV we use the explicit drop queue. */
13543                                 MLX5_ASSERT(priv->drop_queue.hrxq);
13544                                 dv->actions[n++] =
13545                                                 priv->drop_queue.hrxq->action;
13546 #endif
13547                         }
13548                 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
13549                            !dv_h->rix_sample && !dv_h->rix_dest_array)) {
13550                         struct mlx5_hrxq *hrxq;
13551                         uint32_t hrxq_idx;
13552
13553                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc,
13554                                                     &hrxq_idx);
13555                         if (!hrxq) {
13556                                 rte_flow_error_set
13557                                         (error, rte_errno,
13558                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13559                                          "cannot get hash queue");
13560                                 goto error;
13561                         }
13562                         dh->rix_hrxq = hrxq_idx;
13563                         dv->actions[n++] = hrxq->action;
13564                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13565                         struct mlx5_hrxq *hrxq = NULL;
13566                         uint32_t hrxq_idx;
13567
13568                         hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev,
13569                                                 rss_desc->shared_rss,
13570                                                 dev_flow->hash_fields);
13571                         if (hrxq_idx)
13572                                 hrxq = mlx5_ipool_get
13573                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
13574                                          hrxq_idx);
13575                         if (!hrxq) {
13576                                 rte_flow_error_set
13577                                         (error, rte_errno,
13578                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13579                                          "cannot get hash queue");
13580                                 goto error;
13581                         }
13582                         dh->rix_srss = rss_desc->shared_rss;
13583                         dv->actions[n++] = hrxq->action;
13584                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
13585                         if (!priv->sh->default_miss_action) {
13586                                 rte_flow_error_set
13587                                         (error, rte_errno,
13588                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13589                                          "default miss action not be created.");
13590                                 goto error;
13591                         }
13592                         dv->actions[n++] = priv->sh->default_miss_action;
13593                 }
13594                 misc_mask = flow_dv_matcher_enable(dv->value.buf);
13595                 __flow_dv_adjust_buf_size(&dv->value.size, misc_mask);
13596                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
13597                                                (void *)&dv->value, n,
13598                                                dv->actions, &dh->drv_flow);
13599                 if (err) {
13600                         rte_flow_error_set
13601                                 (error, errno,
13602                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
13603                                 NULL,
13604                                 (!priv->config.allow_duplicate_pattern &&
13605                                 errno == EEXIST) ?
13606                                 "duplicating pattern is not allowed" :
13607                                 "hardware refuses to create flow");
13608                         goto error;
13609                 }
13610                 if (priv->vmwa_context &&
13611                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
13612                         /*
13613                          * The rule contains the VLAN pattern.
13614                          * For VF we are going to create VLAN
13615                          * interface to make hypervisor set correct
13616                          * e-Switch vport context.
13617                          */
13618                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
13619                 }
13620         }
13621         return 0;
13622 error:
13623         err = rte_errno; /* Save rte_errno before cleanup. */
13624         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
13625                        handle_idx, dh, next) {
13626                 /* hrxq is union, don't clear it if the flag is not set. */
13627                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
13628                         mlx5_hrxq_release(dev, dh->rix_hrxq);
13629                         dh->rix_hrxq = 0;
13630                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13631                         dh->rix_srss = 0;
13632                 }
13633                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
13634                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
13635         }
13636         rte_errno = err; /* Restore rte_errno. */
13637         return -rte_errno;
13638 }
13639
13640 void
13641 flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
13642                           struct mlx5_list_entry *entry)
13643 {
13644         struct mlx5_flow_dv_matcher *resource = container_of(entry,
13645                                                              typeof(*resource),
13646                                                              entry);
13647
13648         claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
13649         mlx5_free(resource);
13650 }
13651
13652 /**
13653  * Release the flow matcher.
13654  *
13655  * @param dev
13656  *   Pointer to Ethernet device.
13657  * @param port_id
13658  *   Index to port ID action resource.
13659  *
13660  * @return
13661  *   1 while a reference on it exists, 0 when freed.
13662  */
13663 static int
13664 flow_dv_matcher_release(struct rte_eth_dev *dev,
13665                         struct mlx5_flow_handle *handle)
13666 {
13667         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
13668         struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl,
13669                                                             typeof(*tbl), tbl);
13670         int ret;
13671
13672         MLX5_ASSERT(matcher->matcher_object);
13673         ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
13674         flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
13675         return ret;
13676 }
13677
13678 /**
13679  * Release encap_decap resource.
13680  *
13681  * @param list
13682  *   Pointer to the hash list.
13683  * @param entry
13684  *   Pointer to exist resource entry object.
13685  */
13686 void
13687 flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
13688                               struct mlx5_hlist_entry *entry)
13689 {
13690         struct mlx5_dev_ctx_shared *sh = list->ctx;
13691         struct mlx5_flow_dv_encap_decap_resource *res =
13692                                        container_of(entry, typeof(*res), entry);
13693
13694         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13695         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
13696 }
13697
13698 /**
13699  * Release an encap/decap resource.
13700  *
13701  * @param dev
13702  *   Pointer to Ethernet device.
13703  * @param encap_decap_idx
13704  *   Index of encap decap resource.
13705  *
13706  * @return
13707  *   1 while a reference on it exists, 0 when freed.
13708  */
13709 static int
13710 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
13711                                      uint32_t encap_decap_idx)
13712 {
13713         struct mlx5_priv *priv = dev->data->dev_private;
13714         struct mlx5_flow_dv_encap_decap_resource *resource;
13715
13716         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
13717                                   encap_decap_idx);
13718         if (!resource)
13719                 return 0;
13720         MLX5_ASSERT(resource->action);
13721         return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
13722 }
13723
13724 /**
13725  * Release an jump to table action resource.
13726  *
13727  * @param dev
13728  *   Pointer to Ethernet device.
13729  * @param rix_jump
13730  *   Index to the jump action resource.
13731  *
13732  * @return
13733  *   1 while a reference on it exists, 0 when freed.
13734  */
13735 static int
13736 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
13737                                   uint32_t rix_jump)
13738 {
13739         struct mlx5_priv *priv = dev->data->dev_private;
13740         struct mlx5_flow_tbl_data_entry *tbl_data;
13741
13742         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
13743                                   rix_jump);
13744         if (!tbl_data)
13745                 return 0;
13746         return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl);
13747 }
13748
13749 void
13750 flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused,
13751                          struct mlx5_hlist_entry *entry)
13752 {
13753         struct mlx5_flow_dv_modify_hdr_resource *res =
13754                 container_of(entry, typeof(*res), entry);
13755
13756         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13757         mlx5_free(entry);
13758 }
13759
13760 /**
13761  * Release a modify-header resource.
13762  *
13763  * @param dev
13764  *   Pointer to Ethernet device.
13765  * @param handle
13766  *   Pointer to mlx5_flow_handle.
13767  *
13768  * @return
13769  *   1 while a reference on it exists, 0 when freed.
13770  */
13771 static int
13772 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
13773                                     struct mlx5_flow_handle *handle)
13774 {
13775         struct mlx5_priv *priv = dev->data->dev_private;
13776         struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr;
13777
13778         MLX5_ASSERT(entry->action);
13779         return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry);
13780 }
13781
13782 void
13783 flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13784 {
13785         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13786         struct mlx5_flow_dv_port_id_action_resource *resource =
13787                                   container_of(entry, typeof(*resource), entry);
13788
13789         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
13790         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
13791 }
13792
13793 /**
13794  * Release port ID action resource.
13795  *
13796  * @param dev
13797  *   Pointer to Ethernet device.
13798  * @param handle
13799  *   Pointer to mlx5_flow_handle.
13800  *
13801  * @return
13802  *   1 while a reference on it exists, 0 when freed.
13803  */
13804 static int
13805 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
13806                                         uint32_t port_id)
13807 {
13808         struct mlx5_priv *priv = dev->data->dev_private;
13809         struct mlx5_flow_dv_port_id_action_resource *resource;
13810
13811         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
13812         if (!resource)
13813                 return 0;
13814         MLX5_ASSERT(resource->action);
13815         return mlx5_list_unregister(priv->sh->port_id_action_list,
13816                                     &resource->entry);
13817 }
13818
13819 /**
13820  * Release shared RSS action resource.
13821  *
13822  * @param dev
13823  *   Pointer to Ethernet device.
13824  * @param srss
13825  *   Shared RSS action index.
13826  */
13827 static void
13828 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
13829 {
13830         struct mlx5_priv *priv = dev->data->dev_private;
13831         struct mlx5_shared_action_rss *shared_rss;
13832
13833         shared_rss = mlx5_ipool_get
13834                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss);
13835         __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
13836 }
13837
13838 void
13839 flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13840 {
13841         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13842         struct mlx5_flow_dv_push_vlan_action_resource *resource =
13843                         container_of(entry, typeof(*resource), entry);
13844
13845         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
13846         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
13847 }
13848
13849 /**
13850  * Release push vlan action resource.
13851  *
13852  * @param dev
13853  *   Pointer to Ethernet device.
13854  * @param handle
13855  *   Pointer to mlx5_flow_handle.
13856  *
13857  * @return
13858  *   1 while a reference on it exists, 0 when freed.
13859  */
13860 static int
13861 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
13862                                           struct mlx5_flow_handle *handle)
13863 {
13864         struct mlx5_priv *priv = dev->data->dev_private;
13865         struct mlx5_flow_dv_push_vlan_action_resource *resource;
13866         uint32_t idx = handle->dvh.rix_push_vlan;
13867
13868         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
13869         if (!resource)
13870                 return 0;
13871         MLX5_ASSERT(resource->action);
13872         return mlx5_list_unregister(priv->sh->push_vlan_action_list,
13873                                     &resource->entry);
13874 }
13875
13876 /**
13877  * Release the fate resource.
13878  *
13879  * @param dev
13880  *   Pointer to Ethernet device.
13881  * @param handle
13882  *   Pointer to mlx5_flow_handle.
13883  */
13884 static void
13885 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
13886                                struct mlx5_flow_handle *handle)
13887 {
13888         if (!handle->rix_fate)
13889                 return;
13890         switch (handle->fate_action) {
13891         case MLX5_FLOW_FATE_QUEUE:
13892                 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array)
13893                         mlx5_hrxq_release(dev, handle->rix_hrxq);
13894                 break;
13895         case MLX5_FLOW_FATE_JUMP:
13896                 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump);
13897                 break;
13898         case MLX5_FLOW_FATE_PORT_ID:
13899                 flow_dv_port_id_action_resource_release(dev,
13900                                 handle->rix_port_id_action);
13901                 break;
13902         default:
13903                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
13904                 break;
13905         }
13906         handle->rix_fate = 0;
13907 }
13908
13909 void
13910 flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
13911                          struct mlx5_list_entry *entry)
13912 {
13913         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
13914                                                               typeof(*resource),
13915                                                               entry);
13916         struct rte_eth_dev *dev = resource->dev;
13917         struct mlx5_priv *priv = dev->data->dev_private;
13918
13919         if (resource->verbs_action)
13920                 claim_zero(mlx5_flow_os_destroy_flow_action
13921                                                       (resource->verbs_action));
13922         if (resource->normal_path_tbl)
13923                 flow_dv_tbl_resource_release(MLX5_SH(dev),
13924                                              resource->normal_path_tbl);
13925         flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
13926         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
13927         DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
13928 }
13929
13930 /**
13931  * Release an sample resource.
13932  *
13933  * @param dev
13934  *   Pointer to Ethernet device.
13935  * @param handle
13936  *   Pointer to mlx5_flow_handle.
13937  *
13938  * @return
13939  *   1 while a reference on it exists, 0 when freed.
13940  */
13941 static int
13942 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
13943                                      struct mlx5_flow_handle *handle)
13944 {
13945         struct mlx5_priv *priv = dev->data->dev_private;
13946         struct mlx5_flow_dv_sample_resource *resource;
13947
13948         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
13949                                   handle->dvh.rix_sample);
13950         if (!resource)
13951                 return 0;
13952         MLX5_ASSERT(resource->verbs_action);
13953         return mlx5_list_unregister(priv->sh->sample_action_list,
13954                                     &resource->entry);
13955 }
13956
13957 void
13958 flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
13959                              struct mlx5_list_entry *entry)
13960 {
13961         struct mlx5_flow_dv_dest_array_resource *resource =
13962                         container_of(entry, typeof(*resource), entry);
13963         struct rte_eth_dev *dev = resource->dev;
13964         struct mlx5_priv *priv = dev->data->dev_private;
13965         uint32_t i = 0;
13966
13967         MLX5_ASSERT(resource->action);
13968         if (resource->action)
13969                 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
13970         for (; i < resource->num_of_dest; i++)
13971                 flow_dv_sample_sub_actions_release(dev,
13972                                                    &resource->sample_idx[i]);
13973         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
13974         DRV_LOG(DEBUG, "destination array resource %p: removed",
13975                 (void *)resource);
13976 }
13977
13978 /**
13979  * Release an destination array resource.
13980  *
13981  * @param dev
13982  *   Pointer to Ethernet device.
13983  * @param handle
13984  *   Pointer to mlx5_flow_handle.
13985  *
13986  * @return
13987  *   1 while a reference on it exists, 0 when freed.
13988  */
13989 static int
13990 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
13991                                     struct mlx5_flow_handle *handle)
13992 {
13993         struct mlx5_priv *priv = dev->data->dev_private;
13994         struct mlx5_flow_dv_dest_array_resource *resource;
13995
13996         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
13997                                   handle->dvh.rix_dest_array);
13998         if (!resource)
13999                 return 0;
14000         MLX5_ASSERT(resource->action);
14001         return mlx5_list_unregister(priv->sh->dest_array_list,
14002                                     &resource->entry);
14003 }
14004
14005 static void
14006 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev)
14007 {
14008         struct mlx5_priv *priv = dev->data->dev_private;
14009         struct mlx5_dev_ctx_shared *sh = priv->sh;
14010         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
14011                                 sh->geneve_tlv_option_resource;
14012         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
14013         if (geneve_opt_resource) {
14014                 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1,
14015                                          __ATOMIC_RELAXED))) {
14016                         claim_zero(mlx5_devx_cmd_destroy
14017                                         (geneve_opt_resource->obj));
14018                         mlx5_free(sh->geneve_tlv_option_resource);
14019                         sh->geneve_tlv_option_resource = NULL;
14020                 }
14021         }
14022         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
14023 }
14024
14025 /**
14026  * Remove the flow from the NIC but keeps it in memory.
14027  * Lock free, (mutex should be acquired by caller).
14028  *
14029  * @param[in] dev
14030  *   Pointer to Ethernet device.
14031  * @param[in, out] flow
14032  *   Pointer to flow structure.
14033  */
14034 static void
14035 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
14036 {
14037         struct mlx5_flow_handle *dh;
14038         uint32_t handle_idx;
14039         struct mlx5_priv *priv = dev->data->dev_private;
14040
14041         if (!flow)
14042                 return;
14043         handle_idx = flow->dev_handles;
14044         while (handle_idx) {
14045                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14046                                     handle_idx);
14047                 if (!dh)
14048                         return;
14049                 if (dh->drv_flow) {
14050                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
14051                         dh->drv_flow = NULL;
14052                 }
14053                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
14054                         flow_dv_fate_resource_release(dev, dh);
14055                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
14056                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
14057                 handle_idx = dh->next.next;
14058         }
14059 }
14060
14061 /**
14062  * Remove the flow from the NIC and the memory.
14063  * Lock free, (mutex should be acquired by caller).
14064  *
14065  * @param[in] dev
14066  *   Pointer to the Ethernet device structure.
14067  * @param[in, out] flow
14068  *   Pointer to flow structure.
14069  */
14070 static void
14071 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
14072 {
14073         struct mlx5_flow_handle *dev_handle;
14074         struct mlx5_priv *priv = dev->data->dev_private;
14075         struct mlx5_flow_meter_info *fm = NULL;
14076         uint32_t srss = 0;
14077
14078         if (!flow)
14079                 return;
14080         flow_dv_remove(dev, flow);
14081         if (flow->counter) {
14082                 flow_dv_counter_free(dev, flow->counter);
14083                 flow->counter = 0;
14084         }
14085         if (flow->meter) {
14086                 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
14087                 if (fm)
14088                         mlx5_flow_meter_detach(priv, fm);
14089                 flow->meter = 0;
14090         }
14091         /* Keep the current age handling by default. */
14092         if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct)
14093                 flow_dv_aso_ct_release(dev, flow->ct);
14094         else if (flow->age)
14095                 flow_dv_aso_age_release(dev, flow->age);
14096         if (flow->geneve_tlv_option) {
14097                 flow_dv_geneve_tlv_option_resource_release(dev);
14098                 flow->geneve_tlv_option = 0;
14099         }
14100         while (flow->dev_handles) {
14101                 uint32_t tmp_idx = flow->dev_handles;
14102
14103                 dev_handle = mlx5_ipool_get(priv->sh->ipool
14104                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
14105                 if (!dev_handle)
14106                         return;
14107                 flow->dev_handles = dev_handle->next.next;
14108                 if (dev_handle->dvh.matcher)
14109                         flow_dv_matcher_release(dev, dev_handle);
14110                 if (dev_handle->dvh.rix_sample)
14111                         flow_dv_sample_resource_release(dev, dev_handle);
14112                 if (dev_handle->dvh.rix_dest_array)
14113                         flow_dv_dest_array_resource_release(dev, dev_handle);
14114                 if (dev_handle->dvh.rix_encap_decap)
14115                         flow_dv_encap_decap_resource_release(dev,
14116                                 dev_handle->dvh.rix_encap_decap);
14117                 if (dev_handle->dvh.modify_hdr)
14118                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
14119                 if (dev_handle->dvh.rix_push_vlan)
14120                         flow_dv_push_vlan_action_resource_release(dev,
14121                                                                   dev_handle);
14122                 if (dev_handle->dvh.rix_tag)
14123                         flow_dv_tag_release(dev,
14124                                             dev_handle->dvh.rix_tag);
14125                 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS)
14126                         flow_dv_fate_resource_release(dev, dev_handle);
14127                 else if (!srss)
14128                         srss = dev_handle->rix_srss;
14129                 if (fm && dev_handle->is_meter_flow_id &&
14130                     dev_handle->split_flow_id)
14131                         mlx5_ipool_free(fm->flow_ipool,
14132                                         dev_handle->split_flow_id);
14133                 else if (dev_handle->split_flow_id &&
14134                     !dev_handle->is_meter_flow_id)
14135                         mlx5_ipool_free(priv->sh->ipool
14136                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
14137                                         dev_handle->split_flow_id);
14138                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14139                            tmp_idx);
14140         }
14141         if (srss)
14142                 flow_dv_shared_rss_action_release(dev, srss);
14143 }
14144
14145 /**
14146  * Release array of hash RX queue objects.
14147  * Helper function.
14148  *
14149  * @param[in] dev
14150  *   Pointer to the Ethernet device structure.
14151  * @param[in, out] hrxqs
14152  *   Array of hash RX queue objects.
14153  *
14154  * @return
14155  *   Total number of references to hash RX queue objects in *hrxqs* array
14156  *   after this operation.
14157  */
14158 static int
14159 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
14160                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
14161 {
14162         size_t i;
14163         int remaining = 0;
14164
14165         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
14166                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
14167
14168                 if (!ret)
14169                         (*hrxqs)[i] = 0;
14170                 remaining += ret;
14171         }
14172         return remaining;
14173 }
14174
14175 /**
14176  * Release all hash RX queue objects representing shared RSS action.
14177  *
14178  * @param[in] dev
14179  *   Pointer to the Ethernet device structure.
14180  * @param[in, out] action
14181  *   Shared RSS action to remove hash RX queue objects from.
14182  *
14183  * @return
14184  *   Total number of references to hash RX queue objects stored in *action*
14185  *   after this operation.
14186  *   Expected to be 0 if no external references held.
14187  */
14188 static int
14189 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
14190                                  struct mlx5_shared_action_rss *shared_rss)
14191 {
14192         return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq);
14193 }
14194
14195 /**
14196  * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to
14197  * user input.
14198  *
14199  * Only one hash value is available for one L3+L4 combination:
14200  * for example:
14201  * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and
14202  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
14203  * same slot in mlx5_rss_hash_fields.
14204  *
14205  * @param[in] rss
14206  *   Pointer to the shared action RSS conf.
14207  * @param[in, out] hash_field
14208  *   hash_field variable needed to be adjusted.
14209  *
14210  * @return
14211  *   void
14212  */
14213 static void
14214 __flow_dv_action_rss_l34_hash_adjust(struct mlx5_shared_action_rss *rss,
14215                                      uint64_t *hash_field)
14216 {
14217         uint64_t rss_types = rss->origin.types;
14218
14219         switch (*hash_field & ~IBV_RX_HASH_INNER) {
14220         case MLX5_RSS_HASH_IPV4:
14221                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
14222                         *hash_field &= ~MLX5_RSS_HASH_IPV4;
14223                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14224                                 *hash_field |= IBV_RX_HASH_DST_IPV4;
14225                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14226                                 *hash_field |= IBV_RX_HASH_SRC_IPV4;
14227                         else
14228                                 *hash_field |= MLX5_RSS_HASH_IPV4;
14229                 }
14230                 return;
14231         case MLX5_RSS_HASH_IPV6:
14232                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
14233                         *hash_field &= ~MLX5_RSS_HASH_IPV6;
14234                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14235                                 *hash_field |= IBV_RX_HASH_DST_IPV6;
14236                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14237                                 *hash_field |= IBV_RX_HASH_SRC_IPV6;
14238                         else
14239                                 *hash_field |= MLX5_RSS_HASH_IPV6;
14240                 }
14241                 return;
14242         case MLX5_RSS_HASH_IPV4_UDP:
14243                 /* fall-through. */
14244         case MLX5_RSS_HASH_IPV6_UDP:
14245                 if (rss_types & ETH_RSS_UDP) {
14246                         *hash_field &= ~MLX5_UDP_IBV_RX_HASH;
14247                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14248                                 *hash_field |= IBV_RX_HASH_DST_PORT_UDP;
14249                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14250                                 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP;
14251                         else
14252                                 *hash_field |= MLX5_UDP_IBV_RX_HASH;
14253                 }
14254                 return;
14255         case MLX5_RSS_HASH_IPV4_TCP:
14256                 /* fall-through. */
14257         case MLX5_RSS_HASH_IPV6_TCP:
14258                 if (rss_types & ETH_RSS_TCP) {
14259                         *hash_field &= ~MLX5_TCP_IBV_RX_HASH;
14260                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14261                                 *hash_field |= IBV_RX_HASH_DST_PORT_TCP;
14262                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14263                                 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP;
14264                         else
14265                                 *hash_field |= MLX5_TCP_IBV_RX_HASH;
14266                 }
14267                 return;
14268         default:
14269                 return;
14270         }
14271 }
14272
14273 /**
14274  * Setup shared RSS action.
14275  * Prepare set of hash RX queue objects sufficient to handle all valid
14276  * hash_fields combinations (see enum ibv_rx_hash_fields).
14277  *
14278  * @param[in] dev
14279  *   Pointer to the Ethernet device structure.
14280  * @param[in] action_idx
14281  *   Shared RSS action ipool index.
14282  * @param[in, out] action
14283  *   Partially initialized shared RSS action.
14284  * @param[out] error
14285  *   Perform verbose error reporting if not NULL. Initialized in case of
14286  *   error only.
14287  *
14288  * @return
14289  *   0 on success, otherwise negative errno value.
14290  */
14291 static int
14292 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
14293                            uint32_t action_idx,
14294                            struct mlx5_shared_action_rss *shared_rss,
14295                            struct rte_flow_error *error)
14296 {
14297         struct mlx5_flow_rss_desc rss_desc = { 0 };
14298         size_t i;
14299         int err;
14300
14301         if (mlx5_ind_table_obj_setup(dev, shared_rss->ind_tbl)) {
14302                 return rte_flow_error_set(error, rte_errno,
14303                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14304                                           "cannot setup indirection table");
14305         }
14306         memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN);
14307         rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
14308         rss_desc.const_q = shared_rss->origin.queue;
14309         rss_desc.queue_num = shared_rss->origin.queue_num;
14310         /* Set non-zero value to indicate a shared RSS. */
14311         rss_desc.shared_rss = action_idx;
14312         rss_desc.ind_tbl = shared_rss->ind_tbl;
14313         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
14314                 uint32_t hrxq_idx;
14315                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
14316                 int tunnel = 0;
14317
14318                 __flow_dv_action_rss_l34_hash_adjust(shared_rss, &hash_fields);
14319                 if (shared_rss->origin.level > 1) {
14320                         hash_fields |= IBV_RX_HASH_INNER;
14321                         tunnel = 1;
14322                 }
14323                 rss_desc.tunnel = tunnel;
14324                 rss_desc.hash_fields = hash_fields;
14325                 hrxq_idx = mlx5_hrxq_get(dev, &rss_desc);
14326                 if (!hrxq_idx) {
14327                         rte_flow_error_set
14328                                 (error, rte_errno,
14329                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14330                                  "cannot get hash queue");
14331                         goto error_hrxq_new;
14332                 }
14333                 err = __flow_dv_action_rss_hrxq_set
14334                         (shared_rss, hash_fields, hrxq_idx);
14335                 MLX5_ASSERT(!err);
14336         }
14337         return 0;
14338 error_hrxq_new:
14339         err = rte_errno;
14340         __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14341         if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true))
14342                 shared_rss->ind_tbl = NULL;
14343         rte_errno = err;
14344         return -rte_errno;
14345 }
14346
14347 /**
14348  * Create shared RSS action.
14349  *
14350  * @param[in] dev
14351  *   Pointer to the Ethernet device structure.
14352  * @param[in] conf
14353  *   Shared action configuration.
14354  * @param[in] rss
14355  *   RSS action specification used to create shared action.
14356  * @param[out] error
14357  *   Perform verbose error reporting if not NULL. Initialized in case of
14358  *   error only.
14359  *
14360  * @return
14361  *   A valid shared action ID in case of success, 0 otherwise and
14362  *   rte_errno is set.
14363  */
14364 static uint32_t
14365 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
14366                             const struct rte_flow_indir_action_conf *conf,
14367                             const struct rte_flow_action_rss *rss,
14368                             struct rte_flow_error *error)
14369 {
14370         struct mlx5_priv *priv = dev->data->dev_private;
14371         struct mlx5_shared_action_rss *shared_rss = NULL;
14372         void *queue = NULL;
14373         struct rte_flow_action_rss *origin;
14374         const uint8_t *rss_key;
14375         uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
14376         uint32_t idx;
14377
14378         RTE_SET_USED(conf);
14379         queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14380                             0, SOCKET_ID_ANY);
14381         shared_rss = mlx5_ipool_zmalloc
14382                          (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx);
14383         if (!shared_rss || !queue) {
14384                 rte_flow_error_set(error, ENOMEM,
14385                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14386                                    "cannot allocate resource memory");
14387                 goto error_rss_init;
14388         }
14389         if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) {
14390                 rte_flow_error_set(error, E2BIG,
14391                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14392                                    "rss action number out of range");
14393                 goto error_rss_init;
14394         }
14395         shared_rss->ind_tbl = mlx5_malloc(MLX5_MEM_ZERO,
14396                                           sizeof(*shared_rss->ind_tbl),
14397                                           0, SOCKET_ID_ANY);
14398         if (!shared_rss->ind_tbl) {
14399                 rte_flow_error_set(error, ENOMEM,
14400                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14401                                    "cannot allocate resource memory");
14402                 goto error_rss_init;
14403         }
14404         memcpy(queue, rss->queue, queue_size);
14405         shared_rss->ind_tbl->queues = queue;
14406         shared_rss->ind_tbl->queues_n = rss->queue_num;
14407         origin = &shared_rss->origin;
14408         origin->func = rss->func;
14409         origin->level = rss->level;
14410         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
14411         origin->types = !rss->types ? ETH_RSS_IP : rss->types;
14412         /* NULL RSS key indicates default RSS key. */
14413         rss_key = !rss->key ? rss_hash_default_key : rss->key;
14414         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
14415         origin->key = &shared_rss->key[0];
14416         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
14417         origin->queue = queue;
14418         origin->queue_num = rss->queue_num;
14419         if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error))
14420                 goto error_rss_init;
14421         rte_spinlock_init(&shared_rss->action_rss_sl);
14422         __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14423         rte_spinlock_lock(&priv->shared_act_sl);
14424         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14425                      &priv->rss_shared_actions, idx, shared_rss, next);
14426         rte_spinlock_unlock(&priv->shared_act_sl);
14427         return idx;
14428 error_rss_init:
14429         if (shared_rss) {
14430                 if (shared_rss->ind_tbl)
14431                         mlx5_free(shared_rss->ind_tbl);
14432                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14433                                 idx);
14434         }
14435         if (queue)
14436                 mlx5_free(queue);
14437         return 0;
14438 }
14439
14440 /**
14441  * Destroy the shared RSS action.
14442  * Release related hash RX queue objects.
14443  *
14444  * @param[in] dev
14445  *   Pointer to the Ethernet device structure.
14446  * @param[in] idx
14447  *   The shared RSS action object ID to be removed.
14448  * @param[out] error
14449  *   Perform verbose error reporting if not NULL. Initialized in case of
14450  *   error only.
14451  *
14452  * @return
14453  *   0 on success, otherwise negative errno value.
14454  */
14455 static int
14456 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx,
14457                              struct rte_flow_error *error)
14458 {
14459         struct mlx5_priv *priv = dev->data->dev_private;
14460         struct mlx5_shared_action_rss *shared_rss =
14461             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14462         uint32_t old_refcnt = 1;
14463         int remaining;
14464         uint16_t *queue = NULL;
14465
14466         if (!shared_rss)
14467                 return rte_flow_error_set(error, EINVAL,
14468                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14469                                           "invalid shared action");
14470         remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14471         if (remaining)
14472                 return rte_flow_error_set(error, EBUSY,
14473                                           RTE_FLOW_ERROR_TYPE_ACTION,
14474                                           NULL,
14475                                           "shared rss hrxq has references");
14476         if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt,
14477                                          0, 0, __ATOMIC_ACQUIRE,
14478                                          __ATOMIC_RELAXED))
14479                 return rte_flow_error_set(error, EBUSY,
14480                                           RTE_FLOW_ERROR_TYPE_ACTION,
14481                                           NULL,
14482                                           "shared rss has references");
14483         queue = shared_rss->ind_tbl->queues;
14484         remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true);
14485         if (remaining)
14486                 return rte_flow_error_set(error, EBUSY,
14487                                           RTE_FLOW_ERROR_TYPE_ACTION,
14488                                           NULL,
14489                                           "shared rss indirection table has"
14490                                           " references");
14491         mlx5_free(queue);
14492         rte_spinlock_lock(&priv->shared_act_sl);
14493         ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14494                      &priv->rss_shared_actions, idx, shared_rss, next);
14495         rte_spinlock_unlock(&priv->shared_act_sl);
14496         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14497                         idx);
14498         return 0;
14499 }
14500
14501 /**
14502  * Create indirect action, lock free,
14503  * (mutex should be acquired by caller).
14504  * Dispatcher for action type specific call.
14505  *
14506  * @param[in] dev
14507  *   Pointer to the Ethernet device structure.
14508  * @param[in] conf
14509  *   Shared action configuration.
14510  * @param[in] action
14511  *   Action specification used to create indirect action.
14512  * @param[out] error
14513  *   Perform verbose error reporting if not NULL. Initialized in case of
14514  *   error only.
14515  *
14516  * @return
14517  *   A valid shared action handle in case of success, NULL otherwise and
14518  *   rte_errno is set.
14519  */
14520 static struct rte_flow_action_handle *
14521 flow_dv_action_create(struct rte_eth_dev *dev,
14522                       const struct rte_flow_indir_action_conf *conf,
14523                       const struct rte_flow_action *action,
14524                       struct rte_flow_error *err)
14525 {
14526         struct mlx5_priv *priv = dev->data->dev_private;
14527         uint32_t age_idx = 0;
14528         uint32_t idx = 0;
14529         uint32_t ret = 0;
14530
14531         switch (action->type) {
14532         case RTE_FLOW_ACTION_TYPE_RSS:
14533                 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err);
14534                 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS <<
14535                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14536                 break;
14537         case RTE_FLOW_ACTION_TYPE_AGE:
14538                 age_idx = flow_dv_aso_age_alloc(dev, err);
14539                 if (!age_idx) {
14540                         ret = -rte_errno;
14541                         break;
14542                 }
14543                 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
14544                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
14545                 flow_dv_aso_age_params_init(dev, age_idx,
14546                                         ((const struct rte_flow_action_age *)
14547                                                 action->conf)->context ?
14548                                         ((const struct rte_flow_action_age *)
14549                                                 action->conf)->context :
14550                                         (void *)(uintptr_t)idx,
14551                                         ((const struct rte_flow_action_age *)
14552                                                 action->conf)->timeout);
14553                 ret = age_idx;
14554                 break;
14555         case RTE_FLOW_ACTION_TYPE_COUNT:
14556                 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL);
14557                 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT <<
14558                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14559                 break;
14560         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
14561                 ret = flow_dv_translate_create_conntrack(dev, action->conf,
14562                                                          err);
14563                 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret);
14564                 break;
14565         default:
14566                 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
14567                                    NULL, "action type not supported");
14568                 break;
14569         }
14570         return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL;
14571 }
14572
14573 /**
14574  * Destroy the indirect action.
14575  * Release action related resources on the NIC and the memory.
14576  * Lock free, (mutex should be acquired by caller).
14577  * Dispatcher for action type specific call.
14578  *
14579  * @param[in] dev
14580  *   Pointer to the Ethernet device structure.
14581  * @param[in] handle
14582  *   The indirect action object handle to be removed.
14583  * @param[out] error
14584  *   Perform verbose error reporting if not NULL. Initialized in case of
14585  *   error only.
14586  *
14587  * @return
14588  *   0 on success, otherwise negative errno value.
14589  */
14590 static int
14591 flow_dv_action_destroy(struct rte_eth_dev *dev,
14592                        struct rte_flow_action_handle *handle,
14593                        struct rte_flow_error *error)
14594 {
14595         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14596         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14597         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14598         struct mlx5_flow_counter *cnt;
14599         uint32_t no_flow_refcnt = 1;
14600         int ret;
14601
14602         switch (type) {
14603         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14604                 return __flow_dv_action_rss_release(dev, idx, error);
14605         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
14606                 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL);
14607                 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt,
14608                                                  &no_flow_refcnt, 1, false,
14609                                                  __ATOMIC_ACQUIRE,
14610                                                  __ATOMIC_RELAXED))
14611                         return rte_flow_error_set(error, EBUSY,
14612                                                   RTE_FLOW_ERROR_TYPE_ACTION,
14613                                                   NULL,
14614                                                   "Indirect count action has references");
14615                 flow_dv_counter_free(dev, idx);
14616                 return 0;
14617         case MLX5_INDIRECT_ACTION_TYPE_AGE:
14618                 ret = flow_dv_aso_age_release(dev, idx);
14619                 if (ret)
14620                         /*
14621                          * In this case, the last flow has a reference will
14622                          * actually release the age action.
14623                          */
14624                         DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was"
14625                                 " released with references %d.", idx, ret);
14626                 return 0;
14627         case MLX5_INDIRECT_ACTION_TYPE_CT:
14628                 ret = flow_dv_aso_ct_release(dev, idx);
14629                 if (ret < 0)
14630                         return ret;
14631                 if (ret > 0)
14632                         DRV_LOG(DEBUG, "Connection tracking object %u still "
14633                                 "has references %d.", idx, ret);
14634                 return 0;
14635         default:
14636                 return rte_flow_error_set(error, ENOTSUP,
14637                                           RTE_FLOW_ERROR_TYPE_ACTION,
14638                                           NULL,
14639                                           "action type not supported");
14640         }
14641 }
14642
14643 /**
14644  * Updates in place shared RSS action configuration.
14645  *
14646  * @param[in] dev
14647  *   Pointer to the Ethernet device structure.
14648  * @param[in] idx
14649  *   The shared RSS action object ID to be updated.
14650  * @param[in] action_conf
14651  *   RSS action specification used to modify *shared_rss*.
14652  * @param[out] error
14653  *   Perform verbose error reporting if not NULL. Initialized in case of
14654  *   error only.
14655  *
14656  * @return
14657  *   0 on success, otherwise negative errno value.
14658  * @note: currently only support update of RSS queues.
14659  */
14660 static int
14661 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx,
14662                             const struct rte_flow_action_rss *action_conf,
14663                             struct rte_flow_error *error)
14664 {
14665         struct mlx5_priv *priv = dev->data->dev_private;
14666         struct mlx5_shared_action_rss *shared_rss =
14667             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14668         int ret = 0;
14669         void *queue = NULL;
14670         uint16_t *queue_old = NULL;
14671         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
14672
14673         if (!shared_rss)
14674                 return rte_flow_error_set(error, EINVAL,
14675                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14676                                           "invalid shared action to update");
14677         if (priv->obj_ops.ind_table_modify == NULL)
14678                 return rte_flow_error_set(error, ENOTSUP,
14679                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14680                                           "cannot modify indirection table");
14681         queue = mlx5_malloc(MLX5_MEM_ZERO,
14682                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14683                             0, SOCKET_ID_ANY);
14684         if (!queue)
14685                 return rte_flow_error_set(error, ENOMEM,
14686                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14687                                           NULL,
14688                                           "cannot allocate resource memory");
14689         memcpy(queue, action_conf->queue, queue_size);
14690         MLX5_ASSERT(shared_rss->ind_tbl);
14691         rte_spinlock_lock(&shared_rss->action_rss_sl);
14692         queue_old = shared_rss->ind_tbl->queues;
14693         ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl,
14694                                         queue, action_conf->queue_num, true);
14695         if (ret) {
14696                 mlx5_free(queue);
14697                 ret = rte_flow_error_set(error, rte_errno,
14698                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14699                                           "cannot update indirection table");
14700         } else {
14701                 mlx5_free(queue_old);
14702                 shared_rss->origin.queue = queue;
14703                 shared_rss->origin.queue_num = action_conf->queue_num;
14704         }
14705         rte_spinlock_unlock(&shared_rss->action_rss_sl);
14706         return ret;
14707 }
14708
14709 /*
14710  * Updates in place conntrack context or direction.
14711  * Context update should be synchronized.
14712  *
14713  * @param[in] dev
14714  *   Pointer to the Ethernet device structure.
14715  * @param[in] idx
14716  *   The conntrack object ID to be updated.
14717  * @param[in] update
14718  *   Pointer to the structure of information to update.
14719  * @param[out] error
14720  *   Perform verbose error reporting if not NULL. Initialized in case of
14721  *   error only.
14722  *
14723  * @return
14724  *   0 on success, otherwise negative errno value.
14725  */
14726 static int
14727 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx,
14728                            const struct rte_flow_modify_conntrack *update,
14729                            struct rte_flow_error *error)
14730 {
14731         struct mlx5_priv *priv = dev->data->dev_private;
14732         struct mlx5_aso_ct_action *ct;
14733         const struct rte_flow_action_conntrack *new_prf;
14734         int ret = 0;
14735         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
14736         uint32_t dev_idx;
14737
14738         if (PORT_ID(priv) != owner)
14739                 return rte_flow_error_set(error, EACCES,
14740                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14741                                           NULL,
14742                                           "CT object owned by another port");
14743         dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
14744         ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
14745         if (!ct->refcnt)
14746                 return rte_flow_error_set(error, ENOMEM,
14747                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14748                                           NULL,
14749                                           "CT object is inactive");
14750         new_prf = &update->new_ct;
14751         if (update->direction)
14752                 ct->is_original = !!new_prf->is_original_dir;
14753         if (update->state) {
14754                 /* Only validate the profile when it needs to be updated. */
14755                 ret = mlx5_validate_action_ct(dev, new_prf, error);
14756                 if (ret)
14757                         return ret;
14758                 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf);
14759                 if (ret)
14760                         return rte_flow_error_set(error, EIO,
14761                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14762                                         NULL,
14763                                         "Failed to send CT context update WQE");
14764                 /* Block until ready or a failure. */
14765                 ret = mlx5_aso_ct_available(priv->sh, ct);
14766                 if (ret)
14767                         rte_flow_error_set(error, rte_errno,
14768                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14769                                            NULL,
14770                                            "Timeout to get the CT update");
14771         }
14772         return ret;
14773 }
14774
14775 /**
14776  * Updates in place shared action configuration, lock free,
14777  * (mutex should be acquired by caller).
14778  *
14779  * @param[in] dev
14780  *   Pointer to the Ethernet device structure.
14781  * @param[in] handle
14782  *   The indirect action object handle to be updated.
14783  * @param[in] update
14784  *   Action specification used to modify the action pointed by *handle*.
14785  *   *update* could be of same type with the action pointed by the *handle*
14786  *   handle argument, or some other structures like a wrapper, depending on
14787  *   the indirect action type.
14788  * @param[out] error
14789  *   Perform verbose error reporting if not NULL. Initialized in case of
14790  *   error only.
14791  *
14792  * @return
14793  *   0 on success, otherwise negative errno value.
14794  */
14795 static int
14796 flow_dv_action_update(struct rte_eth_dev *dev,
14797                         struct rte_flow_action_handle *handle,
14798                         const void *update,
14799                         struct rte_flow_error *err)
14800 {
14801         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14802         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14803         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14804         const void *action_conf;
14805
14806         switch (type) {
14807         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14808                 action_conf = ((const struct rte_flow_action *)update)->conf;
14809                 return __flow_dv_action_rss_update(dev, idx, action_conf, err);
14810         case MLX5_INDIRECT_ACTION_TYPE_CT:
14811                 return __flow_dv_action_ct_update(dev, idx, update, err);
14812         default:
14813                 return rte_flow_error_set(err, ENOTSUP,
14814                                           RTE_FLOW_ERROR_TYPE_ACTION,
14815                                           NULL,
14816                                           "action type update not supported");
14817         }
14818 }
14819
14820 /**
14821  * Destroy the meter sub policy table rules.
14822  * Lock free, (mutex should be acquired by caller).
14823  *
14824  * @param[in] dev
14825  *   Pointer to Ethernet device.
14826  * @param[in] sub_policy
14827  *   Pointer to meter sub policy table.
14828  */
14829 static void
14830 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
14831                              struct mlx5_flow_meter_sub_policy *sub_policy)
14832 {
14833         struct mlx5_priv *priv = dev->data->dev_private;
14834         struct mlx5_flow_tbl_data_entry *tbl;
14835         struct mlx5_flow_meter_policy *policy = sub_policy->main_policy;
14836         struct mlx5_flow_meter_info *next_fm;
14837         struct mlx5_sub_policy_color_rule *color_rule;
14838         void *tmp;
14839         uint32_t i;
14840
14841         for (i = 0; i < RTE_COLORS; i++) {
14842                 next_fm = NULL;
14843                 if (i == RTE_COLOR_GREEN && policy &&
14844                     policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
14845                         next_fm = mlx5_flow_meter_find(priv,
14846                                         policy->act_cnt[i].next_mtr_id, NULL);
14847                 TAILQ_FOREACH_SAFE(color_rule, &sub_policy->color_rules[i],
14848                                    next_port, tmp) {
14849                         claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
14850                         tbl = container_of(color_rule->matcher->tbl,
14851                                         typeof(*tbl), tbl);
14852                         mlx5_list_unregister(tbl->matchers,
14853                                                 &color_rule->matcher->entry);
14854                         TAILQ_REMOVE(&sub_policy->color_rules[i],
14855                                         color_rule, next_port);
14856                         mlx5_free(color_rule);
14857                         if (next_fm)
14858                                 mlx5_flow_meter_detach(priv, next_fm);
14859                 }
14860         }
14861         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
14862                 if (sub_policy->rix_hrxq[i]) {
14863                         if (policy && !policy->is_hierarchy)
14864                                 mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]);
14865                         sub_policy->rix_hrxq[i] = 0;
14866                 }
14867                 if (sub_policy->jump_tbl[i]) {
14868                         flow_dv_tbl_resource_release(MLX5_SH(dev),
14869                         sub_policy->jump_tbl[i]);
14870                         sub_policy->jump_tbl[i] = NULL;
14871                 }
14872         }
14873         if (sub_policy->tbl_rsc) {
14874                 flow_dv_tbl_resource_release(MLX5_SH(dev),
14875                         sub_policy->tbl_rsc);
14876                 sub_policy->tbl_rsc = NULL;
14877         }
14878 }
14879
14880 /**
14881  * Destroy policy rules, lock free,
14882  * (mutex should be acquired by caller).
14883  * Dispatcher for action type specific call.
14884  *
14885  * @param[in] dev
14886  *   Pointer to the Ethernet device structure.
14887  * @param[in] mtr_policy
14888  *   Meter policy struct.
14889  */
14890 static void
14891 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev,
14892                       struct mlx5_flow_meter_policy *mtr_policy)
14893 {
14894         uint32_t i, j;
14895         struct mlx5_flow_meter_sub_policy *sub_policy;
14896         uint16_t sub_policy_num;
14897
14898         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
14899                 sub_policy_num = (mtr_policy->sub_policy_num >>
14900                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
14901                         MLX5_MTR_SUB_POLICY_NUM_MASK;
14902                 for (j = 0; j < sub_policy_num; j++) {
14903                         sub_policy = mtr_policy->sub_policys[i][j];
14904                         if (sub_policy)
14905                                 __flow_dv_destroy_sub_policy_rules
14906                                                 (dev, sub_policy);
14907                 }
14908         }
14909 }
14910
14911 /**
14912  * Destroy policy action, lock free,
14913  * (mutex should be acquired by caller).
14914  * Dispatcher for action type specific call.
14915  *
14916  * @param[in] dev
14917  *   Pointer to the Ethernet device structure.
14918  * @param[in] mtr_policy
14919  *   Meter policy struct.
14920  */
14921 static void
14922 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
14923                       struct mlx5_flow_meter_policy *mtr_policy)
14924 {
14925         struct rte_flow_action *rss_action;
14926         struct mlx5_flow_handle dev_handle;
14927         uint32_t i, j;
14928
14929         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
14930                 if (mtr_policy->act_cnt[i].rix_mark) {
14931                         flow_dv_tag_release(dev,
14932                                 mtr_policy->act_cnt[i].rix_mark);
14933                         mtr_policy->act_cnt[i].rix_mark = 0;
14934                 }
14935                 if (mtr_policy->act_cnt[i].modify_hdr) {
14936                         dev_handle.dvh.modify_hdr =
14937                                 mtr_policy->act_cnt[i].modify_hdr;
14938                         flow_dv_modify_hdr_resource_release(dev, &dev_handle);
14939                 }
14940                 switch (mtr_policy->act_cnt[i].fate_action) {
14941                 case MLX5_FLOW_FATE_SHARED_RSS:
14942                         rss_action = mtr_policy->act_cnt[i].rss;
14943                         mlx5_free(rss_action);
14944                         break;
14945                 case MLX5_FLOW_FATE_PORT_ID:
14946                         if (mtr_policy->act_cnt[i].rix_port_id_action) {
14947                                 flow_dv_port_id_action_resource_release(dev,
14948                                 mtr_policy->act_cnt[i].rix_port_id_action);
14949                                 mtr_policy->act_cnt[i].rix_port_id_action = 0;
14950                         }
14951                         break;
14952                 case MLX5_FLOW_FATE_DROP:
14953                 case MLX5_FLOW_FATE_JUMP:
14954                         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
14955                                 mtr_policy->act_cnt[i].dr_jump_action[j] =
14956                                                 NULL;
14957                         break;
14958                 default:
14959                         /*Queue action do nothing*/
14960                         break;
14961                 }
14962         }
14963         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
14964                 mtr_policy->dr_drop_action[j] = NULL;
14965 }
14966
14967 /**
14968  * Create policy action per domain, lock free,
14969  * (mutex should be acquired by caller).
14970  * Dispatcher for action type specific call.
14971  *
14972  * @param[in] dev
14973  *   Pointer to the Ethernet device structure.
14974  * @param[in] mtr_policy
14975  *   Meter policy struct.
14976  * @param[in] action
14977  *   Action specification used to create meter actions.
14978  * @param[out] error
14979  *   Perform verbose error reporting if not NULL. Initialized in case of
14980  *   error only.
14981  *
14982  * @return
14983  *   0 on success, otherwise negative errno value.
14984  */
14985 static int
14986 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
14987                         struct mlx5_flow_meter_policy *mtr_policy,
14988                         const struct rte_flow_action *actions[RTE_COLORS],
14989                         enum mlx5_meter_domain domain,
14990                         struct rte_mtr_error *error)
14991 {
14992         struct mlx5_priv *priv = dev->data->dev_private;
14993         struct rte_flow_error flow_err;
14994         const struct rte_flow_action *act;
14995         uint64_t action_flags = 0;
14996         struct mlx5_flow_handle dh;
14997         struct mlx5_flow dev_flow;
14998         struct mlx5_flow_dv_port_id_action_resource port_id_action;
14999         int i, ret;
15000         uint8_t egress, transfer;
15001         struct mlx5_meter_policy_action_container *act_cnt = NULL;
15002         union {
15003                 struct mlx5_flow_dv_modify_hdr_resource res;
15004                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
15005                             sizeof(struct mlx5_modification_cmd) *
15006                             (MLX5_MAX_MODIFY_NUM + 1)];
15007         } mhdr_dummy;
15008         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
15009
15010         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15011         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15012         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
15013         memset(&dev_flow, 0, sizeof(struct mlx5_flow));
15014         memset(&port_id_action, 0,
15015                 sizeof(struct mlx5_flow_dv_port_id_action_resource));
15016         memset(mhdr_res, 0, sizeof(*mhdr_res));
15017         mhdr_res->ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
15018                                         egress ?
15019                                         MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
15020                                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
15021         dev_flow.handle = &dh;
15022         dev_flow.dv.port_id_action = &port_id_action;
15023         dev_flow.external = true;
15024         for (i = 0; i < RTE_COLORS; i++) {
15025                 if (i < MLX5_MTR_RTE_COLORS)
15026                         act_cnt = &mtr_policy->act_cnt[i];
15027                 for (act = actions[i];
15028                         act && act->type != RTE_FLOW_ACTION_TYPE_END;
15029                         act++) {
15030                         switch (act->type) {
15031                         case RTE_FLOW_ACTION_TYPE_MARK:
15032                         {
15033                                 uint32_t tag_be = mlx5_flow_mark_set
15034                                         (((const struct rte_flow_action_mark *)
15035                                         (act->conf))->id);
15036
15037                                 if (i >= MLX5_MTR_RTE_COLORS)
15038                                         return -rte_mtr_error_set(error,
15039                                           ENOTSUP,
15040                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15041                                           NULL,
15042                                           "cannot create policy "
15043                                           "mark action for this color");
15044                                 dev_flow.handle->mark = 1;
15045                                 if (flow_dv_tag_resource_register(dev, tag_be,
15046                                                   &dev_flow, &flow_err))
15047                                         return -rte_mtr_error_set(error,
15048                                         ENOTSUP,
15049                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15050                                         NULL,
15051                                         "cannot setup policy mark action");
15052                                 MLX5_ASSERT(dev_flow.dv.tag_resource);
15053                                 act_cnt->rix_mark =
15054                                         dev_flow.handle->dvh.rix_tag;
15055                                 action_flags |= MLX5_FLOW_ACTION_MARK;
15056                                 break;
15057                         }
15058                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
15059                                 if (i >= MLX5_MTR_RTE_COLORS)
15060                                         return -rte_mtr_error_set(error,
15061                                           ENOTSUP,
15062                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15063                                           NULL,
15064                                           "cannot create policy "
15065                                           "set tag action for this color");
15066                                 if (flow_dv_convert_action_set_tag
15067                                 (dev, mhdr_res,
15068                                 (const struct rte_flow_action_set_tag *)
15069                                 act->conf,  &flow_err))
15070                                         return -rte_mtr_error_set(error,
15071                                         ENOTSUP,
15072                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15073                                         NULL, "cannot convert policy "
15074                                         "set tag action");
15075                                 if (!mhdr_res->actions_num)
15076                                         return -rte_mtr_error_set(error,
15077                                         ENOTSUP,
15078                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15079                                         NULL, "cannot find policy "
15080                                         "set tag action");
15081                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
15082                                 break;
15083                         case RTE_FLOW_ACTION_TYPE_DROP:
15084                         {
15085                                 struct mlx5_flow_mtr_mng *mtrmng =
15086                                                 priv->sh->mtrmng;
15087                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15088
15089                                 /*
15090                                  * Create the drop table with
15091                                  * METER DROP level.
15092                                  */
15093                                 if (!mtrmng->drop_tbl[domain]) {
15094                                         mtrmng->drop_tbl[domain] =
15095                                         flow_dv_tbl_resource_get(dev,
15096                                         MLX5_FLOW_TABLE_LEVEL_METER,
15097                                         egress, transfer, false, NULL, 0,
15098                                         0, MLX5_MTR_TABLE_ID_DROP, &flow_err);
15099                                         if (!mtrmng->drop_tbl[domain])
15100                                                 return -rte_mtr_error_set
15101                                         (error, ENOTSUP,
15102                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15103                                         NULL,
15104                                         "Failed to create meter drop table");
15105                                 }
15106                                 tbl_data = container_of
15107                                 (mtrmng->drop_tbl[domain],
15108                                 struct mlx5_flow_tbl_data_entry, tbl);
15109                                 if (i < MLX5_MTR_RTE_COLORS) {
15110                                         act_cnt->dr_jump_action[domain] =
15111                                                 tbl_data->jump.action;
15112                                         act_cnt->fate_action =
15113                                                 MLX5_FLOW_FATE_DROP;
15114                                 }
15115                                 if (i == RTE_COLOR_RED)
15116                                         mtr_policy->dr_drop_action[domain] =
15117                                                 tbl_data->jump.action;
15118                                 action_flags |= MLX5_FLOW_ACTION_DROP;
15119                                 break;
15120                         }
15121                         case RTE_FLOW_ACTION_TYPE_QUEUE:
15122                         {
15123                                 if (i >= MLX5_MTR_RTE_COLORS)
15124                                         return -rte_mtr_error_set(error,
15125                                         ENOTSUP,
15126                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15127                                         NULL, "cannot create policy "
15128                                         "fate queue for this color");
15129                                 act_cnt->queue =
15130                                 ((const struct rte_flow_action_queue *)
15131                                         (act->conf))->index;
15132                                 act_cnt->fate_action =
15133                                         MLX5_FLOW_FATE_QUEUE;
15134                                 dev_flow.handle->fate_action =
15135                                         MLX5_FLOW_FATE_QUEUE;
15136                                 mtr_policy->is_queue = 1;
15137                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
15138                                 break;
15139                         }
15140                         case RTE_FLOW_ACTION_TYPE_RSS:
15141                         {
15142                                 int rss_size;
15143
15144                                 if (i >= MLX5_MTR_RTE_COLORS)
15145                                         return -rte_mtr_error_set(error,
15146                                           ENOTSUP,
15147                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15148                                           NULL,
15149                                           "cannot create policy "
15150                                           "rss action for this color");
15151                                 /*
15152                                  * Save RSS conf into policy struct
15153                                  * for translate stage.
15154                                  */
15155                                 rss_size = (int)rte_flow_conv
15156                                         (RTE_FLOW_CONV_OP_ACTION,
15157                                         NULL, 0, act, &flow_err);
15158                                 if (rss_size <= 0)
15159                                         return -rte_mtr_error_set(error,
15160                                           ENOTSUP,
15161                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15162                                           NULL, "Get the wrong "
15163                                           "rss action struct size");
15164                                 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO,
15165                                                 rss_size, 0, SOCKET_ID_ANY);
15166                                 if (!act_cnt->rss)
15167                                         return -rte_mtr_error_set(error,
15168                                           ENOTSUP,
15169                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15170                                           NULL,
15171                                           "Fail to malloc rss action memory");
15172                                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION,
15173                                         act_cnt->rss, rss_size,
15174                                         act, &flow_err);
15175                                 if (ret < 0)
15176                                         return -rte_mtr_error_set(error,
15177                                           ENOTSUP,
15178                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15179                                           NULL, "Fail to save "
15180                                           "rss action into policy struct");
15181                                 act_cnt->fate_action =
15182                                         MLX5_FLOW_FATE_SHARED_RSS;
15183                                 action_flags |= MLX5_FLOW_ACTION_RSS;
15184                                 break;
15185                         }
15186                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
15187                         {
15188                                 struct mlx5_flow_dv_port_id_action_resource
15189                                         port_id_resource;
15190                                 uint32_t port_id = 0;
15191
15192                                 if (i >= MLX5_MTR_RTE_COLORS)
15193                                         return -rte_mtr_error_set(error,
15194                                         ENOTSUP,
15195                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15196                                         NULL, "cannot create policy "
15197                                         "port action for this color");
15198                                 memset(&port_id_resource, 0,
15199                                         sizeof(port_id_resource));
15200                                 if (flow_dv_translate_action_port_id(dev, act,
15201                                                 &port_id, &flow_err))
15202                                         return -rte_mtr_error_set(error,
15203                                         ENOTSUP,
15204                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15205                                         NULL, "cannot translate "
15206                                         "policy port action");
15207                                 port_id_resource.port_id = port_id;
15208                                 if (flow_dv_port_id_action_resource_register
15209                                         (dev, &port_id_resource,
15210                                         &dev_flow, &flow_err))
15211                                         return -rte_mtr_error_set(error,
15212                                         ENOTSUP,
15213                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15214                                         NULL, "cannot setup "
15215                                         "policy port action");
15216                                 act_cnt->rix_port_id_action =
15217                                         dev_flow.handle->rix_port_id_action;
15218                                 act_cnt->fate_action =
15219                                         MLX5_FLOW_FATE_PORT_ID;
15220                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
15221                                 break;
15222                         }
15223                         case RTE_FLOW_ACTION_TYPE_JUMP:
15224                         {
15225                                 uint32_t jump_group = 0;
15226                                 uint32_t table = 0;
15227                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15228                                 struct flow_grp_info grp_info = {
15229                                         .external = !!dev_flow.external,
15230                                         .transfer = !!transfer,
15231                                         .fdb_def_rule = !!priv->fdb_def_rule,
15232                                         .std_tbl_fix = 0,
15233                                         .skip_scale = dev_flow.skip_scale &
15234                                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
15235                                 };
15236                                 struct mlx5_flow_meter_sub_policy *sub_policy =
15237                                 mtr_policy->sub_policys[domain][0];
15238
15239                                 if (i >= MLX5_MTR_RTE_COLORS)
15240                                         return -rte_mtr_error_set(error,
15241                                           ENOTSUP,
15242                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15243                                           NULL,
15244                                           "cannot create policy "
15245                                           "jump action for this color");
15246                                 jump_group =
15247                                 ((const struct rte_flow_action_jump *)
15248                                                         act->conf)->group;
15249                                 if (mlx5_flow_group_to_table(dev, NULL,
15250                                                        jump_group,
15251                                                        &table,
15252                                                        &grp_info, &flow_err))
15253                                         return -rte_mtr_error_set(error,
15254                                         ENOTSUP,
15255                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15256                                         NULL, "cannot setup "
15257                                         "policy jump action");
15258                                 sub_policy->jump_tbl[i] =
15259                                 flow_dv_tbl_resource_get(dev,
15260                                         table, egress,
15261                                         transfer,
15262                                         !!dev_flow.external,
15263                                         NULL, jump_group, 0,
15264                                         0, &flow_err);
15265                                 if
15266                                 (!sub_policy->jump_tbl[i])
15267                                         return  -rte_mtr_error_set(error,
15268                                         ENOTSUP,
15269                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15270                                         NULL, "cannot create jump action.");
15271                                 tbl_data = container_of
15272                                 (sub_policy->jump_tbl[i],
15273                                 struct mlx5_flow_tbl_data_entry, tbl);
15274                                 act_cnt->dr_jump_action[domain] =
15275                                         tbl_data->jump.action;
15276                                 act_cnt->fate_action =
15277                                         MLX5_FLOW_FATE_JUMP;
15278                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
15279                                 break;
15280                         }
15281                         case RTE_FLOW_ACTION_TYPE_METER:
15282                         {
15283                                 const struct rte_flow_action_meter *mtr;
15284                                 struct mlx5_flow_meter_info *next_fm;
15285                                 struct mlx5_flow_meter_policy *next_policy;
15286                                 struct rte_flow_action tag_action;
15287                                 struct mlx5_rte_flow_action_set_tag set_tag;
15288                                 uint32_t next_mtr_idx = 0;
15289
15290                                 mtr = act->conf;
15291                                 next_fm = mlx5_flow_meter_find(priv,
15292                                                         mtr->mtr_id,
15293                                                         &next_mtr_idx);
15294                                 if (!next_fm)
15295                                         return -rte_mtr_error_set(error, EINVAL,
15296                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15297                                                 "Fail to find next meter.");
15298                                 if (next_fm->def_policy)
15299                                         return -rte_mtr_error_set(error, EINVAL,
15300                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15301                                 "Hierarchy only supports termination meter.");
15302                                 next_policy = mlx5_flow_meter_policy_find(dev,
15303                                                 next_fm->policy_id, NULL);
15304                                 MLX5_ASSERT(next_policy);
15305                                 if (next_fm->drop_cnt) {
15306                                         set_tag.id =
15307                                                 (enum modify_reg)
15308                                                 mlx5_flow_get_reg_id(dev,
15309                                                 MLX5_MTR_ID,
15310                                                 0,
15311                                                 (struct rte_flow_error *)error);
15312                                         set_tag.offset = (priv->mtr_reg_share ?
15313                                                 MLX5_MTR_COLOR_BITS : 0);
15314                                         set_tag.length = (priv->mtr_reg_share ?
15315                                                MLX5_MTR_IDLE_BITS_IN_COLOR_REG :
15316                                                MLX5_REG_BITS);
15317                                         set_tag.data = next_mtr_idx;
15318                                         tag_action.type =
15319                                                 (enum rte_flow_action_type)
15320                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
15321                                         tag_action.conf = &set_tag;
15322                                         if (flow_dv_convert_action_set_reg
15323                                                 (mhdr_res, &tag_action,
15324                                                 (struct rte_flow_error *)error))
15325                                                 return -rte_errno;
15326                                         action_flags |=
15327                                                 MLX5_FLOW_ACTION_SET_TAG;
15328                                 }
15329                                 act_cnt->fate_action = MLX5_FLOW_FATE_MTR;
15330                                 act_cnt->next_mtr_id = next_fm->meter_id;
15331                                 act_cnt->next_sub_policy = NULL;
15332                                 mtr_policy->is_hierarchy = 1;
15333                                 mtr_policy->dev = next_policy->dev;
15334                                 action_flags |=
15335                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
15336                                 break;
15337                         }
15338                         default:
15339                                 return -rte_mtr_error_set(error, ENOTSUP,
15340                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15341                                           NULL, "action type not supported");
15342                         }
15343                         if (action_flags & MLX5_FLOW_ACTION_SET_TAG) {
15344                                 /* create modify action if needed. */
15345                                 dev_flow.dv.group = 1;
15346                                 if (flow_dv_modify_hdr_resource_register
15347                                         (dev, mhdr_res, &dev_flow, &flow_err))
15348                                         return -rte_mtr_error_set(error,
15349                                                 ENOTSUP,
15350                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
15351                                                 NULL, "cannot register policy "
15352                                                 "set tag action");
15353                                 act_cnt->modify_hdr =
15354                                         dev_flow.handle->dvh.modify_hdr;
15355                         }
15356                 }
15357         }
15358         return 0;
15359 }
15360
15361 /**
15362  * Create policy action per domain, lock free,
15363  * (mutex should be acquired by caller).
15364  * Dispatcher for action type specific call.
15365  *
15366  * @param[in] dev
15367  *   Pointer to the Ethernet device structure.
15368  * @param[in] mtr_policy
15369  *   Meter policy struct.
15370  * @param[in] action
15371  *   Action specification used to create meter actions.
15372  * @param[out] error
15373  *   Perform verbose error reporting if not NULL. Initialized in case of
15374  *   error only.
15375  *
15376  * @return
15377  *   0 on success, otherwise negative errno value.
15378  */
15379 static int
15380 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev,
15381                       struct mlx5_flow_meter_policy *mtr_policy,
15382                       const struct rte_flow_action *actions[RTE_COLORS],
15383                       struct rte_mtr_error *error)
15384 {
15385         int ret, i;
15386         uint16_t sub_policy_num;
15387
15388         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15389                 sub_policy_num = (mtr_policy->sub_policy_num >>
15390                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15391                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15392                 if (sub_policy_num) {
15393                         ret = __flow_dv_create_domain_policy_acts(dev,
15394                                 mtr_policy, actions,
15395                                 (enum mlx5_meter_domain)i, error);
15396                         if (ret)
15397                                 return ret;
15398                 }
15399         }
15400         return 0;
15401 }
15402
15403 /**
15404  * Query a DV flow rule for its statistics via DevX.
15405  *
15406  * @param[in] dev
15407  *   Pointer to Ethernet device.
15408  * @param[in] cnt_idx
15409  *   Index to the flow counter.
15410  * @param[out] data
15411  *   Data retrieved by the query.
15412  * @param[out] error
15413  *   Perform verbose error reporting if not NULL.
15414  *
15415  * @return
15416  *   0 on success, a negative errno value otherwise and rte_errno is set.
15417  */
15418 static int
15419 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data,
15420                     struct rte_flow_error *error)
15421 {
15422         struct mlx5_priv *priv = dev->data->dev_private;
15423         struct rte_flow_query_count *qc = data;
15424
15425         if (!priv->config.devx)
15426                 return rte_flow_error_set(error, ENOTSUP,
15427                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15428                                           NULL,
15429                                           "counters are not supported");
15430         if (cnt_idx) {
15431                 uint64_t pkts, bytes;
15432                 struct mlx5_flow_counter *cnt;
15433                 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes);
15434
15435                 if (err)
15436                         return rte_flow_error_set(error, -err,
15437                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15438                                         NULL, "cannot read counters");
15439                 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL);
15440                 qc->hits_set = 1;
15441                 qc->bytes_set = 1;
15442                 qc->hits = pkts - cnt->hits;
15443                 qc->bytes = bytes - cnt->bytes;
15444                 if (qc->reset) {
15445                         cnt->hits = pkts;
15446                         cnt->bytes = bytes;
15447                 }
15448                 return 0;
15449         }
15450         return rte_flow_error_set(error, EINVAL,
15451                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15452                                   NULL,
15453                                   "counters are not available");
15454 }
15455
15456 static int
15457 flow_dv_action_query(struct rte_eth_dev *dev,
15458                      const struct rte_flow_action_handle *handle, void *data,
15459                      struct rte_flow_error *error)
15460 {
15461         struct mlx5_age_param *age_param;
15462         struct rte_flow_query_age *resp;
15463         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15464         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15465         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15466         struct mlx5_priv *priv = dev->data->dev_private;
15467         struct mlx5_aso_ct_action *ct;
15468         uint16_t owner;
15469         uint32_t dev_idx;
15470
15471         switch (type) {
15472         case MLX5_INDIRECT_ACTION_TYPE_AGE:
15473                 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params;
15474                 resp = data;
15475                 resp->aged = __atomic_load_n(&age_param->state,
15476                                               __ATOMIC_RELAXED) == AGE_TMOUT ?
15477                                                                           1 : 0;
15478                 resp->sec_since_last_hit_valid = !resp->aged;
15479                 if (resp->sec_since_last_hit_valid)
15480                         resp->sec_since_last_hit = __atomic_load_n
15481                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15482                 return 0;
15483         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
15484                 return flow_dv_query_count(dev, idx, data, error);
15485         case MLX5_INDIRECT_ACTION_TYPE_CT:
15486                 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15487                 if (owner != PORT_ID(priv))
15488                         return rte_flow_error_set(error, EACCES,
15489                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15490                                         NULL,
15491                                         "CT object owned by another port");
15492                 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15493                 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15494                 MLX5_ASSERT(ct);
15495                 if (!ct->refcnt)
15496                         return rte_flow_error_set(error, EFAULT,
15497                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15498                                         NULL,
15499                                         "CT object is inactive");
15500                 ((struct rte_flow_action_conntrack *)data)->peer_port =
15501                                                         ct->peer;
15502                 ((struct rte_flow_action_conntrack *)data)->is_original_dir =
15503                                                         ct->is_original;
15504                 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data))
15505                         return rte_flow_error_set(error, EIO,
15506                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15507                                         NULL,
15508                                         "Failed to query CT context");
15509                 return 0;
15510         default:
15511                 return rte_flow_error_set(error, ENOTSUP,
15512                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15513                                           "action type query not supported");
15514         }
15515 }
15516
15517 /**
15518  * Query a flow rule AGE action for aging information.
15519  *
15520  * @param[in] dev
15521  *   Pointer to Ethernet device.
15522  * @param[in] flow
15523  *   Pointer to the sub flow.
15524  * @param[out] data
15525  *   data retrieved by the query.
15526  * @param[out] error
15527  *   Perform verbose error reporting if not NULL.
15528  *
15529  * @return
15530  *   0 on success, a negative errno value otherwise and rte_errno is set.
15531  */
15532 static int
15533 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
15534                   void *data, struct rte_flow_error *error)
15535 {
15536         struct rte_flow_query_age *resp = data;
15537         struct mlx5_age_param *age_param;
15538
15539         if (flow->age) {
15540                 struct mlx5_aso_age_action *act =
15541                                      flow_aso_age_get_by_idx(dev, flow->age);
15542
15543                 age_param = &act->age_params;
15544         } else if (flow->counter) {
15545                 age_param = flow_dv_counter_idx_get_age(dev, flow->counter);
15546
15547                 if (!age_param || !age_param->timeout)
15548                         return rte_flow_error_set
15549                                         (error, EINVAL,
15550                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15551                                          NULL, "cannot read age data");
15552         } else {
15553                 return rte_flow_error_set(error, EINVAL,
15554                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15555                                           NULL, "age data not available");
15556         }
15557         resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
15558                                      AGE_TMOUT ? 1 : 0;
15559         resp->sec_since_last_hit_valid = !resp->aged;
15560         if (resp->sec_since_last_hit_valid)
15561                 resp->sec_since_last_hit = __atomic_load_n
15562                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15563         return 0;
15564 }
15565
15566 /**
15567  * Query a flow.
15568  *
15569  * @see rte_flow_query()
15570  * @see rte_flow_ops
15571  */
15572 static int
15573 flow_dv_query(struct rte_eth_dev *dev,
15574               struct rte_flow *flow __rte_unused,
15575               const struct rte_flow_action *actions __rte_unused,
15576               void *data __rte_unused,
15577               struct rte_flow_error *error __rte_unused)
15578 {
15579         int ret = -EINVAL;
15580
15581         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
15582                 switch (actions->type) {
15583                 case RTE_FLOW_ACTION_TYPE_VOID:
15584                         break;
15585                 case RTE_FLOW_ACTION_TYPE_COUNT:
15586                         ret = flow_dv_query_count(dev, flow->counter, data,
15587                                                   error);
15588                         break;
15589                 case RTE_FLOW_ACTION_TYPE_AGE:
15590                         ret = flow_dv_query_age(dev, flow, data, error);
15591                         break;
15592                 default:
15593                         return rte_flow_error_set(error, ENOTSUP,
15594                                                   RTE_FLOW_ERROR_TYPE_ACTION,
15595                                                   actions,
15596                                                   "action not supported");
15597                 }
15598         }
15599         return ret;
15600 }
15601
15602 /**
15603  * Destroy the meter table set.
15604  * Lock free, (mutex should be acquired by caller).
15605  *
15606  * @param[in] dev
15607  *   Pointer to Ethernet device.
15608  * @param[in] fm
15609  *   Meter information table.
15610  */
15611 static void
15612 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
15613                         struct mlx5_flow_meter_info *fm)
15614 {
15615         struct mlx5_priv *priv = dev->data->dev_private;
15616         int i;
15617
15618         if (!fm || !priv->config.dv_flow_en)
15619                 return;
15620         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15621                 if (fm->drop_rule[i]) {
15622                         claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
15623                         fm->drop_rule[i] = NULL;
15624                 }
15625         }
15626 }
15627
15628 static void
15629 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
15630 {
15631         struct mlx5_priv *priv = dev->data->dev_private;
15632         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15633         struct mlx5_flow_tbl_data_entry *tbl;
15634         int i, j;
15635
15636         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15637                 if (mtrmng->def_rule[i]) {
15638                         claim_zero(mlx5_flow_os_destroy_flow
15639                                         (mtrmng->def_rule[i]));
15640                         mtrmng->def_rule[i] = NULL;
15641                 }
15642                 if (mtrmng->def_matcher[i]) {
15643                         tbl = container_of(mtrmng->def_matcher[i]->tbl,
15644                                 struct mlx5_flow_tbl_data_entry, tbl);
15645                         mlx5_list_unregister(tbl->matchers,
15646                                              &mtrmng->def_matcher[i]->entry);
15647                         mtrmng->def_matcher[i] = NULL;
15648                 }
15649                 for (j = 0; j < MLX5_REG_BITS; j++) {
15650                         if (mtrmng->drop_matcher[i][j]) {
15651                                 tbl =
15652                                 container_of(mtrmng->drop_matcher[i][j]->tbl,
15653                                              struct mlx5_flow_tbl_data_entry,
15654                                              tbl);
15655                                 mlx5_list_unregister(tbl->matchers,
15656                                             &mtrmng->drop_matcher[i][j]->entry);
15657                                 mtrmng->drop_matcher[i][j] = NULL;
15658                         }
15659                 }
15660                 if (mtrmng->drop_tbl[i]) {
15661                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15662                                 mtrmng->drop_tbl[i]);
15663                         mtrmng->drop_tbl[i] = NULL;
15664                 }
15665         }
15666 }
15667
15668 /* Number of meter flow actions, count and jump or count and drop. */
15669 #define METER_ACTIONS 2
15670
15671 static void
15672 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
15673                               enum mlx5_meter_domain domain)
15674 {
15675         struct mlx5_priv *priv = dev->data->dev_private;
15676         struct mlx5_flow_meter_def_policy *def_policy =
15677                         priv->sh->mtrmng->def_policy[domain];
15678
15679         __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
15680         mlx5_free(def_policy);
15681         priv->sh->mtrmng->def_policy[domain] = NULL;
15682 }
15683
15684 /**
15685  * Destroy the default policy table set.
15686  *
15687  * @param[in] dev
15688  *   Pointer to Ethernet device.
15689  */
15690 static void
15691 flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
15692 {
15693         struct mlx5_priv *priv = dev->data->dev_private;
15694         int i;
15695
15696         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
15697                 if (priv->sh->mtrmng->def_policy[i])
15698                         __flow_dv_destroy_domain_def_policy(dev,
15699                                         (enum mlx5_meter_domain)i);
15700         priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
15701 }
15702
15703 static int
15704 __flow_dv_create_policy_flow(struct rte_eth_dev *dev,
15705                         uint32_t color_reg_c_idx,
15706                         enum rte_color color, void *matcher_object,
15707                         int actions_n, void *actions,
15708                         bool match_src_port, const struct rte_flow_item *item,
15709                         void **rule, const struct rte_flow_attr *attr)
15710 {
15711         int ret;
15712         struct mlx5_flow_dv_match_params value = {
15713                 .size = sizeof(value.buf),
15714         };
15715         struct mlx5_flow_dv_match_params matcher = {
15716                 .size = sizeof(matcher.buf),
15717         };
15718         struct mlx5_priv *priv = dev->data->dev_private;
15719         uint8_t misc_mask;
15720
15721         if (match_src_port && (priv->representor || priv->master)) {
15722                 if (flow_dv_translate_item_port_id(dev, matcher.buf,
15723                                                    value.buf, item, attr)) {
15724                         DRV_LOG(ERR,
15725                         "Failed to create meter policy flow with port.");
15726                         return -1;
15727                 }
15728         }
15729         flow_dv_match_meta_reg(matcher.buf, value.buf,
15730                                 (enum modify_reg)color_reg_c_idx,
15731                                 rte_col_2_mlx5_col(color),
15732                                 UINT32_MAX);
15733         misc_mask = flow_dv_matcher_enable(value.buf);
15734         __flow_dv_adjust_buf_size(&value.size, misc_mask);
15735         ret = mlx5_flow_os_create_flow(matcher_object,
15736                         (void *)&value, actions_n, actions, rule);
15737         if (ret) {
15738                 DRV_LOG(ERR, "Failed to create meter policy flow.");
15739                 return -1;
15740         }
15741         return 0;
15742 }
15743
15744 static int
15745 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
15746                         uint32_t color_reg_c_idx,
15747                         uint16_t priority,
15748                         struct mlx5_flow_meter_sub_policy *sub_policy,
15749                         const struct rte_flow_attr *attr,
15750                         bool match_src_port,
15751                         const struct rte_flow_item *item,
15752                         struct mlx5_flow_dv_matcher **policy_matcher,
15753                         struct rte_flow_error *error)
15754 {
15755         struct mlx5_list_entry *entry;
15756         struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
15757         struct mlx5_flow_dv_matcher matcher = {
15758                 .mask = {
15759                         .size = sizeof(matcher.mask.buf),
15760                 },
15761                 .tbl = tbl_rsc,
15762         };
15763         struct mlx5_flow_dv_match_params value = {
15764                 .size = sizeof(value.buf),
15765         };
15766         struct mlx5_flow_cb_ctx ctx = {
15767                 .error = error,
15768                 .data = &matcher,
15769         };
15770         struct mlx5_flow_tbl_data_entry *tbl_data;
15771         struct mlx5_priv *priv = dev->data->dev_private;
15772         uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
15773
15774         if (match_src_port && (priv->representor || priv->master)) {
15775                 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
15776                                                    value.buf, item, attr)) {
15777                         DRV_LOG(ERR,
15778                         "Failed to register meter drop matcher with port.");
15779                         return -1;
15780                 }
15781         }
15782         tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
15783         if (priority < RTE_COLOR_RED)
15784                 flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
15785                         (enum modify_reg)color_reg_c_idx, 0, color_mask);
15786         matcher.priority = priority;
15787         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
15788                                         matcher.mask.size);
15789         entry = mlx5_list_register(tbl_data->matchers, &ctx);
15790         if (!entry) {
15791                 DRV_LOG(ERR, "Failed to register meter drop matcher.");
15792                 return -1;
15793         }
15794         *policy_matcher =
15795                 container_of(entry, struct mlx5_flow_dv_matcher, entry);
15796         return 0;
15797 }
15798
15799 /**
15800  * Create the policy rules per domain.
15801  *
15802  * @param[in] dev
15803  *   Pointer to Ethernet device.
15804  * @param[in] sub_policy
15805  *    Pointer to sub policy table..
15806  * @param[in] egress
15807  *   Direction of the table.
15808  * @param[in] transfer
15809  *   E-Switch or NIC flow.
15810  * @param[in] acts
15811  *   Pointer to policy action list per color.
15812  *
15813  * @return
15814  *   0 on success, -1 otherwise.
15815  */
15816 static int
15817 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
15818                 struct mlx5_flow_meter_sub_policy *sub_policy,
15819                 uint8_t egress, uint8_t transfer, bool match_src_port,
15820                 struct mlx5_meter_policy_acts acts[RTE_COLORS])
15821 {
15822         struct mlx5_priv *priv = dev->data->dev_private;
15823         struct rte_flow_error flow_err;
15824         uint32_t color_reg_c_idx;
15825         struct rte_flow_attr attr = {
15826                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
15827                 .priority = 0,
15828                 .ingress = 0,
15829                 .egress = !!egress,
15830                 .transfer = !!transfer,
15831                 .reserved = 0,
15832         };
15833         int i;
15834         int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
15835         struct mlx5_sub_policy_color_rule *color_rule;
15836
15837         if (ret < 0)
15838                 return -1;
15839         /* Create policy table with POLICY level. */
15840         if (!sub_policy->tbl_rsc)
15841                 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
15842                                 MLX5_FLOW_TABLE_LEVEL_POLICY,
15843                                 egress, transfer, false, NULL, 0, 0,
15844                                 sub_policy->idx, &flow_err);
15845         if (!sub_policy->tbl_rsc) {
15846                 DRV_LOG(ERR,
15847                         "Failed to create meter sub policy table.");
15848                 return -1;
15849         }
15850         /* Prepare matchers. */
15851         color_reg_c_idx = ret;
15852         for (i = 0; i < RTE_COLORS; i++) {
15853                 TAILQ_INIT(&sub_policy->color_rules[i]);
15854                 if (i == RTE_COLOR_YELLOW || !acts[i].actions_n)
15855                         continue;
15856                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
15857                                 sizeof(struct mlx5_sub_policy_color_rule),
15858                                 0, SOCKET_ID_ANY);
15859                 if (!color_rule) {
15860                         DRV_LOG(ERR, "No memory to create color rule.");
15861                         goto err_exit;
15862                 }
15863                 color_rule->src_port = priv->representor_id;
15864                 attr.priority = i;
15865                 /* Create matchers for Color. */
15866                 if (__flow_dv_create_policy_matcher(dev,
15867                                 color_reg_c_idx, i, sub_policy, &attr,
15868                                 (i != RTE_COLOR_RED ? match_src_port : false),
15869                                 NULL, &color_rule->matcher, &flow_err)) {
15870                         DRV_LOG(ERR, "Failed to create color matcher.");
15871                         goto err_exit;
15872                 }
15873                 /* Create flow, matching color. */
15874                 if (__flow_dv_create_policy_flow(dev,
15875                                 color_reg_c_idx, (enum rte_color)i,
15876                                 color_rule->matcher->matcher_object,
15877                                 acts[i].actions_n,
15878                                 acts[i].dv_actions,
15879                                 (i != RTE_COLOR_RED ? match_src_port : false),
15880                                 NULL, &color_rule->rule,
15881                                 &attr)) {
15882                         DRV_LOG(ERR, "Failed to create color rule.");
15883                         goto err_exit;
15884                 }
15885                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
15886                                   color_rule, next_port);
15887         }
15888         return 0;
15889 err_exit:
15890         if (color_rule) {
15891                 if (color_rule->rule)
15892                         mlx5_flow_os_destroy_flow(color_rule->rule);
15893                 if (color_rule->matcher) {
15894                         struct mlx5_flow_tbl_data_entry *tbl =
15895                                 container_of(color_rule->matcher->tbl,
15896                                                 typeof(*tbl), tbl);
15897                         mlx5_list_unregister(tbl->matchers,
15898                                                 &color_rule->matcher->entry);
15899                 }
15900                 mlx5_free(color_rule);
15901         }
15902         return -1;
15903 }
15904
15905 static int
15906 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
15907                         struct mlx5_flow_meter_policy *mtr_policy,
15908                         struct mlx5_flow_meter_sub_policy *sub_policy,
15909                         uint32_t domain)
15910 {
15911         struct mlx5_priv *priv = dev->data->dev_private;
15912         struct mlx5_meter_policy_acts acts[RTE_COLORS];
15913         struct mlx5_flow_dv_tag_resource *tag;
15914         struct mlx5_flow_dv_port_id_action_resource *port_action;
15915         struct mlx5_hrxq *hrxq;
15916         struct mlx5_flow_meter_info *next_fm = NULL;
15917         struct mlx5_flow_meter_policy *next_policy;
15918         struct mlx5_flow_meter_sub_policy *next_sub_policy;
15919         struct mlx5_flow_tbl_data_entry *tbl_data;
15920         struct rte_flow_error error;
15921         uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15922         uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15923         bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX);
15924         bool match_src_port = false;
15925         int i;
15926
15927         for (i = 0; i < RTE_COLORS; i++) {
15928                 acts[i].actions_n = 0;
15929                 if (i == RTE_COLOR_YELLOW)
15930                         continue;
15931                 if (i == RTE_COLOR_RED) {
15932                         /* Only support drop on red. */
15933                         acts[i].dv_actions[0] =
15934                         mtr_policy->dr_drop_action[domain];
15935                         acts[i].actions_n = 1;
15936                         continue;
15937                 }
15938                 if (mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
15939                         struct rte_flow_attr attr = {
15940                                 .transfer = transfer
15941                         };
15942
15943                         next_fm = mlx5_flow_meter_find(priv,
15944                                         mtr_policy->act_cnt[i].next_mtr_id,
15945                                         NULL);
15946                         if (!next_fm) {
15947                                 DRV_LOG(ERR,
15948                                         "Failed to get next hierarchy meter.");
15949                                 goto err_exit;
15950                         }
15951                         if (mlx5_flow_meter_attach(priv, next_fm,
15952                                                    &attr, &error)) {
15953                                 DRV_LOG(ERR, "%s", error.message);
15954                                 next_fm = NULL;
15955                                 goto err_exit;
15956                         }
15957                         /* Meter action must be the first for TX. */
15958                         if (mtr_first) {
15959                                 acts[i].dv_actions[acts[i].actions_n] =
15960                                         next_fm->meter_action;
15961                                 acts[i].actions_n++;
15962                         }
15963                 }
15964                 if (mtr_policy->act_cnt[i].rix_mark) {
15965                         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
15966                                         mtr_policy->act_cnt[i].rix_mark);
15967                         if (!tag) {
15968                                 DRV_LOG(ERR, "Failed to find "
15969                                 "mark action for policy.");
15970                                 goto err_exit;
15971                         }
15972                         acts[i].dv_actions[acts[i].actions_n] =
15973                                                 tag->action;
15974                         acts[i].actions_n++;
15975                 }
15976                 if (mtr_policy->act_cnt[i].modify_hdr) {
15977                         acts[i].dv_actions[acts[i].actions_n] =
15978                         mtr_policy->act_cnt[i].modify_hdr->action;
15979                         acts[i].actions_n++;
15980                 }
15981                 if (mtr_policy->act_cnt[i].fate_action) {
15982                         switch (mtr_policy->act_cnt[i].fate_action) {
15983                         case MLX5_FLOW_FATE_PORT_ID:
15984                                 port_action = mlx5_ipool_get
15985                                         (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
15986                                 mtr_policy->act_cnt[i].rix_port_id_action);
15987                                 if (!port_action) {
15988                                         DRV_LOG(ERR, "Failed to find "
15989                                                 "port action for policy.");
15990                                         goto err_exit;
15991                                 }
15992                                 acts[i].dv_actions[acts[i].actions_n] =
15993                                 port_action->action;
15994                                 acts[i].actions_n++;
15995                                 mtr_policy->dev = dev;
15996                                 match_src_port = true;
15997                                 break;
15998                         case MLX5_FLOW_FATE_DROP:
15999                         case MLX5_FLOW_FATE_JUMP:
16000                                 acts[i].dv_actions[acts[i].actions_n] =
16001                                 mtr_policy->act_cnt[i].dr_jump_action[domain];
16002                                 acts[i].actions_n++;
16003                                 break;
16004                         case MLX5_FLOW_FATE_SHARED_RSS:
16005                         case MLX5_FLOW_FATE_QUEUE:
16006                                 hrxq = mlx5_ipool_get
16007                                 (priv->sh->ipool[MLX5_IPOOL_HRXQ],
16008                                 sub_policy->rix_hrxq[i]);
16009                                 if (!hrxq) {
16010                                         DRV_LOG(ERR, "Failed to find "
16011                                                 "queue action for policy.");
16012                                         goto err_exit;
16013                                 }
16014                                 acts[i].dv_actions[acts[i].actions_n] =
16015                                 hrxq->action;
16016                                 acts[i].actions_n++;
16017                                 break;
16018                         case MLX5_FLOW_FATE_MTR:
16019                                 if (!next_fm) {
16020                                         DRV_LOG(ERR,
16021                                                 "No next hierarchy meter.");
16022                                         goto err_exit;
16023                                 }
16024                                 if (!mtr_first) {
16025                                         acts[i].dv_actions[acts[i].actions_n] =
16026                                                         next_fm->meter_action;
16027                                         acts[i].actions_n++;
16028                                 }
16029                                 if (mtr_policy->act_cnt[i].next_sub_policy) {
16030                                         next_sub_policy =
16031                                         mtr_policy->act_cnt[i].next_sub_policy;
16032                                 } else {
16033                                         next_policy =
16034                                                 mlx5_flow_meter_policy_find(dev,
16035                                                 next_fm->policy_id, NULL);
16036                                         MLX5_ASSERT(next_policy);
16037                                         next_sub_policy =
16038                                         next_policy->sub_policys[domain][0];
16039                                 }
16040                                 tbl_data =
16041                                         container_of(next_sub_policy->tbl_rsc,
16042                                         struct mlx5_flow_tbl_data_entry, tbl);
16043                                 acts[i].dv_actions[acts[i].actions_n++] =
16044                                                         tbl_data->jump.action;
16045                                 if (mtr_policy->act_cnt[i].modify_hdr)
16046                                         match_src_port = !!transfer;
16047                                 break;
16048                         default:
16049                                 /*Queue action do nothing*/
16050                                 break;
16051                         }
16052                 }
16053         }
16054         if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
16055                                 egress, transfer, match_src_port, acts)) {
16056                 DRV_LOG(ERR,
16057                 "Failed to create policy rules per domain.");
16058                 goto err_exit;
16059         }
16060         return 0;
16061 err_exit:
16062         if (next_fm)
16063                 mlx5_flow_meter_detach(priv, next_fm);
16064         return -1;
16065 }
16066
16067 /**
16068  * Create the policy rules.
16069  *
16070  * @param[in] dev
16071  *   Pointer to Ethernet device.
16072  * @param[in,out] mtr_policy
16073  *   Pointer to meter policy table.
16074  *
16075  * @return
16076  *   0 on success, -1 otherwise.
16077  */
16078 static int
16079 flow_dv_create_policy_rules(struct rte_eth_dev *dev,
16080                              struct mlx5_flow_meter_policy *mtr_policy)
16081 {
16082         int i;
16083         uint16_t sub_policy_num;
16084
16085         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16086                 sub_policy_num = (mtr_policy->sub_policy_num >>
16087                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
16088                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16089                 if (!sub_policy_num)
16090                         continue;
16091                 /* Prepare actions list and create policy rules. */
16092                 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16093                         mtr_policy->sub_policys[i][0], i)) {
16094                         DRV_LOG(ERR,
16095                         "Failed to create policy action list per domain.");
16096                         return -1;
16097                 }
16098         }
16099         return 0;
16100 }
16101
16102 static int
16103 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
16104 {
16105         struct mlx5_priv *priv = dev->data->dev_private;
16106         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16107         struct mlx5_flow_meter_def_policy *def_policy;
16108         struct mlx5_flow_tbl_resource *jump_tbl;
16109         struct mlx5_flow_tbl_data_entry *tbl_data;
16110         uint8_t egress, transfer;
16111         struct rte_flow_error error;
16112         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16113         int ret;
16114
16115         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16116         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16117         def_policy = mtrmng->def_policy[domain];
16118         if (!def_policy) {
16119                 def_policy = mlx5_malloc(MLX5_MEM_ZERO,
16120                         sizeof(struct mlx5_flow_meter_def_policy),
16121                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
16122                 if (!def_policy) {
16123                         DRV_LOG(ERR, "Failed to alloc "
16124                                         "default policy table.");
16125                         goto def_policy_error;
16126                 }
16127                 mtrmng->def_policy[domain] = def_policy;
16128                 /* Create the meter suffix table with SUFFIX level. */
16129                 jump_tbl = flow_dv_tbl_resource_get(dev,
16130                                 MLX5_FLOW_TABLE_LEVEL_METER,
16131                                 egress, transfer, false, NULL, 0,
16132                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16133                 if (!jump_tbl) {
16134                         DRV_LOG(ERR,
16135                                 "Failed to create meter suffix table.");
16136                         goto def_policy_error;
16137                 }
16138                 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
16139                 tbl_data = container_of(jump_tbl,
16140                                 struct mlx5_flow_tbl_data_entry, tbl);
16141                 def_policy->dr_jump_action[RTE_COLOR_GREEN] =
16142                                                 tbl_data->jump.action;
16143                 acts[RTE_COLOR_GREEN].dv_actions[0] =
16144                                                 tbl_data->jump.action;
16145                 acts[RTE_COLOR_GREEN].actions_n = 1;
16146                 /* Create jump action to the drop table. */
16147                 if (!mtrmng->drop_tbl[domain]) {
16148                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
16149                                 (dev, MLX5_FLOW_TABLE_LEVEL_METER,
16150                                 egress, transfer, false, NULL, 0,
16151                                 0, MLX5_MTR_TABLE_ID_DROP, &error);
16152                         if (!mtrmng->drop_tbl[domain]) {
16153                                 DRV_LOG(ERR, "Failed to create "
16154                                 "meter drop table for default policy.");
16155                                 goto def_policy_error;
16156                         }
16157                 }
16158                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16159                                 struct mlx5_flow_tbl_data_entry, tbl);
16160                 def_policy->dr_jump_action[RTE_COLOR_RED] =
16161                                                 tbl_data->jump.action;
16162                 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
16163                 acts[RTE_COLOR_RED].actions_n = 1;
16164                 /* Create default policy rules. */
16165                 ret = __flow_dv_create_domain_policy_rules(dev,
16166                                         &def_policy->sub_policy,
16167                                         egress, transfer, false, acts);
16168                 if (ret) {
16169                         DRV_LOG(ERR, "Failed to create "
16170                                 "default policy rules.");
16171                                 goto def_policy_error;
16172                 }
16173         }
16174         return 0;
16175 def_policy_error:
16176         __flow_dv_destroy_domain_def_policy(dev,
16177                         (enum mlx5_meter_domain)domain);
16178         return -1;
16179 }
16180
16181 /**
16182  * Create the default policy table set.
16183  *
16184  * @param[in] dev
16185  *   Pointer to Ethernet device.
16186  * @return
16187  *   0 on success, -1 otherwise.
16188  */
16189 static int
16190 flow_dv_create_def_policy(struct rte_eth_dev *dev)
16191 {
16192         struct mlx5_priv *priv = dev->data->dev_private;
16193         int i;
16194
16195         /* Non-termination policy table. */
16196         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16197                 if (!priv->config.dv_esw_en && i == MLX5_MTR_DOMAIN_TRANSFER)
16198                         continue;
16199                 if (__flow_dv_create_domain_def_policy(dev, i)) {
16200                         DRV_LOG(ERR,
16201                         "Failed to create default policy");
16202                         return -1;
16203                 }
16204         }
16205         return 0;
16206 }
16207
16208 /**
16209  * Create the needed meter tables.
16210  * Lock free, (mutex should be acquired by caller).
16211  *
16212  * @param[in] dev
16213  *   Pointer to Ethernet device.
16214  * @param[in] fm
16215  *   Meter information table.
16216  * @param[in] mtr_idx
16217  *   Meter index.
16218  * @param[in] domain_bitmap
16219  *   Domain bitmap.
16220  * @return
16221  *   0 on success, -1 otherwise.
16222  */
16223 static int
16224 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
16225                         struct mlx5_flow_meter_info *fm,
16226                         uint32_t mtr_idx,
16227                         uint8_t domain_bitmap)
16228 {
16229         struct mlx5_priv *priv = dev->data->dev_private;
16230         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16231         struct rte_flow_error error;
16232         struct mlx5_flow_tbl_data_entry *tbl_data;
16233         uint8_t egress, transfer;
16234         void *actions[METER_ACTIONS];
16235         int domain, ret, i;
16236         struct mlx5_flow_counter *cnt;
16237         struct mlx5_flow_dv_match_params value = {
16238                 .size = sizeof(value.buf),
16239         };
16240         struct mlx5_flow_dv_match_params matcher_para = {
16241                 .size = sizeof(matcher_para.buf),
16242         };
16243         int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
16244                                                      0, &error);
16245         uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
16246         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
16247         struct mlx5_list_entry *entry;
16248         struct mlx5_flow_dv_matcher matcher = {
16249                 .mask = {
16250                         .size = sizeof(matcher.mask.buf),
16251                 },
16252         };
16253         struct mlx5_flow_dv_matcher *drop_matcher;
16254         struct mlx5_flow_cb_ctx ctx = {
16255                 .error = &error,
16256                 .data = &matcher,
16257         };
16258         uint8_t misc_mask;
16259
16260         if (!priv->mtr_en || mtr_id_reg_c < 0) {
16261                 rte_errno = ENOTSUP;
16262                 return -1;
16263         }
16264         for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
16265                 if (!(domain_bitmap & (1 << domain)) ||
16266                         (mtrmng->def_rule[domain] && !fm->drop_cnt))
16267                         continue;
16268                 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16269                 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16270                 /* Create the drop table with METER DROP level. */
16271                 if (!mtrmng->drop_tbl[domain]) {
16272                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
16273                                         MLX5_FLOW_TABLE_LEVEL_METER,
16274                                         egress, transfer, false, NULL, 0,
16275                                         0, MLX5_MTR_TABLE_ID_DROP, &error);
16276                         if (!mtrmng->drop_tbl[domain]) {
16277                                 DRV_LOG(ERR, "Failed to create meter drop table.");
16278                                 goto policy_error;
16279                         }
16280                 }
16281                 /* Create default matcher in drop table. */
16282                 matcher.tbl = mtrmng->drop_tbl[domain],
16283                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16284                                 struct mlx5_flow_tbl_data_entry, tbl);
16285                 if (!mtrmng->def_matcher[domain]) {
16286                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16287                                        (enum modify_reg)mtr_id_reg_c,
16288                                        0, 0);
16289                         matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
16290                         matcher.crc = rte_raw_cksum
16291                                         ((const void *)matcher.mask.buf,
16292                                         matcher.mask.size);
16293                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16294                         if (!entry) {
16295                                 DRV_LOG(ERR, "Failed to register meter "
16296                                 "drop default matcher.");
16297                                 goto policy_error;
16298                         }
16299                         mtrmng->def_matcher[domain] = container_of(entry,
16300                         struct mlx5_flow_dv_matcher, entry);
16301                 }
16302                 /* Create default rule in drop table. */
16303                 if (!mtrmng->def_rule[domain]) {
16304                         i = 0;
16305                         actions[i++] = priv->sh->dr_drop_action;
16306                         flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16307                                 (enum modify_reg)mtr_id_reg_c, 0, 0);
16308                         misc_mask = flow_dv_matcher_enable(value.buf);
16309                         __flow_dv_adjust_buf_size(&value.size, misc_mask);
16310                         ret = mlx5_flow_os_create_flow
16311                                 (mtrmng->def_matcher[domain]->matcher_object,
16312                                 (void *)&value, i, actions,
16313                                 &mtrmng->def_rule[domain]);
16314                         if (ret) {
16315                                 DRV_LOG(ERR, "Failed to create meter "
16316                                 "default drop rule for drop table.");
16317                                 goto policy_error;
16318                         }
16319                 }
16320                 if (!fm->drop_cnt)
16321                         continue;
16322                 MLX5_ASSERT(mtrmng->max_mtr_bits);
16323                 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
16324                         /* Create matchers for Drop. */
16325                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16326                                         (enum modify_reg)mtr_id_reg_c, 0,
16327                                         (mtr_id_mask << mtr_id_offset));
16328                         matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
16329                         matcher.crc = rte_raw_cksum
16330                                         ((const void *)matcher.mask.buf,
16331                                         matcher.mask.size);
16332                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16333                         if (!entry) {
16334                                 DRV_LOG(ERR,
16335                                 "Failed to register meter drop matcher.");
16336                                 goto policy_error;
16337                         }
16338                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
16339                                 container_of(entry, struct mlx5_flow_dv_matcher,
16340                                              entry);
16341                 }
16342                 drop_matcher =
16343                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
16344                 /* Create drop rule, matching meter_id only. */
16345                 flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16346                                 (enum modify_reg)mtr_id_reg_c,
16347                                 (mtr_idx << mtr_id_offset), UINT32_MAX);
16348                 i = 0;
16349                 cnt = flow_dv_counter_get_by_idx(dev,
16350                                         fm->drop_cnt, NULL);
16351                 actions[i++] = cnt->action;
16352                 actions[i++] = priv->sh->dr_drop_action;
16353                 misc_mask = flow_dv_matcher_enable(value.buf);
16354                 __flow_dv_adjust_buf_size(&value.size, misc_mask);
16355                 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
16356                                                (void *)&value, i, actions,
16357                                                &fm->drop_rule[domain]);
16358                 if (ret) {
16359                         DRV_LOG(ERR, "Failed to create meter "
16360                                 "drop rule for drop table.");
16361                                 goto policy_error;
16362                 }
16363         }
16364         return 0;
16365 policy_error:
16366         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16367                 if (fm->drop_rule[i]) {
16368                         claim_zero(mlx5_flow_os_destroy_flow
16369                                 (fm->drop_rule[i]));
16370                         fm->drop_rule[i] = NULL;
16371                 }
16372         }
16373         return -1;
16374 }
16375
16376 static struct mlx5_flow_meter_sub_policy *
16377 __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
16378                 struct mlx5_flow_meter_policy *mtr_policy,
16379                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS],
16380                 struct mlx5_flow_meter_sub_policy *next_sub_policy,
16381                 bool *is_reuse)
16382 {
16383         struct mlx5_priv *priv = dev->data->dev_private;
16384         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16385         uint32_t sub_policy_idx = 0;
16386         uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
16387         uint32_t i, j;
16388         struct mlx5_hrxq *hrxq;
16389         struct mlx5_flow_handle dh;
16390         struct mlx5_meter_policy_action_container *act_cnt;
16391         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16392         uint16_t sub_policy_num;
16393
16394         rte_spinlock_lock(&mtr_policy->sl);
16395         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16396                 if (!rss_desc[i])
16397                         continue;
16398                 hrxq_idx[i] = mlx5_hrxq_get(dev, rss_desc[i]);
16399                 if (!hrxq_idx[i]) {
16400                         rte_spinlock_unlock(&mtr_policy->sl);
16401                         return NULL;
16402                 }
16403         }
16404         sub_policy_num = (mtr_policy->sub_policy_num >>
16405                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16406                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16407         for (i = 0; i < sub_policy_num;
16408                 i++) {
16409                 for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) {
16410                         if (rss_desc[j] &&
16411                                 hrxq_idx[j] !=
16412                         mtr_policy->sub_policys[domain][i]->rix_hrxq[j])
16413                                 break;
16414                 }
16415                 if (j >= MLX5_MTR_RTE_COLORS) {
16416                         /*
16417                          * Found the sub policy table with
16418                          * the same queue per color
16419                          */
16420                         rte_spinlock_unlock(&mtr_policy->sl);
16421                         for (j = 0; j < MLX5_MTR_RTE_COLORS; j++)
16422                                 mlx5_hrxq_release(dev, hrxq_idx[j]);
16423                         *is_reuse = true;
16424                         return mtr_policy->sub_policys[domain][i];
16425                 }
16426         }
16427         /* Create sub policy. */
16428         if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
16429                 /* Reuse the first dummy sub_policy*/
16430                 sub_policy = mtr_policy->sub_policys[domain][0];
16431                 sub_policy_idx = sub_policy->idx;
16432         } else {
16433                 sub_policy = mlx5_ipool_zmalloc
16434                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16435                                 &sub_policy_idx);
16436                 if (!sub_policy ||
16437                         sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
16438                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
16439                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
16440                         goto rss_sub_policy_error;
16441                 }
16442                 sub_policy->idx = sub_policy_idx;
16443                 sub_policy->main_policy = mtr_policy;
16444         }
16445         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16446                 if (!rss_desc[i])
16447                         continue;
16448                 sub_policy->rix_hrxq[i] = hrxq_idx[i];
16449                 if (mtr_policy->is_hierarchy) {
16450                         act_cnt = &mtr_policy->act_cnt[i];
16451                         act_cnt->next_sub_policy = next_sub_policy;
16452                         mlx5_hrxq_release(dev, hrxq_idx[i]);
16453                 } else {
16454                         /*
16455                          * Overwrite the last action from
16456                          * RSS action to Queue action.
16457                          */
16458                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
16459                                 hrxq_idx[i]);
16460                         if (!hrxq) {
16461                                 DRV_LOG(ERR, "Failed to create policy hrxq");
16462                                 goto rss_sub_policy_error;
16463                         }
16464                         act_cnt = &mtr_policy->act_cnt[i];
16465                         if (act_cnt->rix_mark || act_cnt->modify_hdr) {
16466                                 memset(&dh, 0, sizeof(struct mlx5_flow_handle));
16467                                 if (act_cnt->rix_mark)
16468                                         dh.mark = 1;
16469                                 dh.fate_action = MLX5_FLOW_FATE_QUEUE;
16470                                 dh.rix_hrxq = hrxq_idx[i];
16471                                 flow_drv_rxq_flags_set(dev, &dh);
16472                         }
16473                 }
16474         }
16475         if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16476                 sub_policy, domain)) {
16477                 DRV_LOG(ERR, "Failed to create policy "
16478                         "rules per domain.");
16479                 goto rss_sub_policy_error;
16480         }
16481         if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16482                 i = (mtr_policy->sub_policy_num >>
16483                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16484                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16485                 mtr_policy->sub_policys[domain][i] = sub_policy;
16486                 i++;
16487                 if (i > MLX5_MTR_RSS_MAX_SUB_POLICY)
16488                         goto rss_sub_policy_error;
16489                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16490                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16491                 mtr_policy->sub_policy_num |=
16492                         (i & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16493                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16494         }
16495         rte_spinlock_unlock(&mtr_policy->sl);
16496         *is_reuse = false;
16497         return sub_policy;
16498 rss_sub_policy_error:
16499         if (sub_policy) {
16500                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16501                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16502                         i = (mtr_policy->sub_policy_num >>
16503                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16504                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16505                         mtr_policy->sub_policys[domain][i] = NULL;
16506                         mlx5_ipool_free
16507                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16508                                         sub_policy->idx);
16509                 }
16510         }
16511         rte_spinlock_unlock(&mtr_policy->sl);
16512         return NULL;
16513 }
16514
16515 /**
16516  * Find the policy table for prefix table with RSS.
16517  *
16518  * @param[in] dev
16519  *   Pointer to Ethernet device.
16520  * @param[in] mtr_policy
16521  *   Pointer to meter policy table.
16522  * @param[in] rss_desc
16523  *   Pointer to rss_desc
16524  * @return
16525  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
16526  */
16527 static struct mlx5_flow_meter_sub_policy *
16528 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
16529                 struct mlx5_flow_meter_policy *mtr_policy,
16530                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
16531 {
16532         struct mlx5_priv *priv = dev->data->dev_private;
16533         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16534         struct mlx5_flow_meter_info *next_fm;
16535         struct mlx5_flow_meter_policy *next_policy;
16536         struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL;
16537         struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM];
16538         struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM];
16539         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16540         bool reuse_sub_policy;
16541         uint32_t i = 0;
16542         uint32_t j = 0;
16543
16544         while (true) {
16545                 /* Iterate hierarchy to get all policies in this hierarchy. */
16546                 policies[i++] = mtr_policy;
16547                 if (!mtr_policy->is_hierarchy)
16548                         break;
16549                 if (i >= MLX5_MTR_CHAIN_MAX_NUM) {
16550                         DRV_LOG(ERR, "Exceed max meter number in hierarchy.");
16551                         return NULL;
16552                 }
16553                 next_fm = mlx5_flow_meter_find(priv,
16554                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16555                 if (!next_fm) {
16556                         DRV_LOG(ERR, "Failed to get next meter in hierarchy.");
16557                         return NULL;
16558                 }
16559                 next_policy =
16560                         mlx5_flow_meter_policy_find(dev, next_fm->policy_id,
16561                                                     NULL);
16562                 MLX5_ASSERT(next_policy);
16563                 mtr_policy = next_policy;
16564         }
16565         while (i) {
16566                 /**
16567                  * From last policy to the first one in hierarchy,
16568                  * create/get the sub policy for each of them.
16569                  */
16570                 sub_policy = __flow_dv_meter_get_rss_sub_policy(dev,
16571                                                         policies[--i],
16572                                                         rss_desc,
16573                                                         next_sub_policy,
16574                                                         &reuse_sub_policy);
16575                 if (!sub_policy) {
16576                         DRV_LOG(ERR, "Failed to get the sub policy.");
16577                         goto err_exit;
16578                 }
16579                 if (!reuse_sub_policy)
16580                         sub_policies[j++] = sub_policy;
16581                 next_sub_policy = sub_policy;
16582         }
16583         return sub_policy;
16584 err_exit:
16585         while (j) {
16586                 uint16_t sub_policy_num;
16587
16588                 sub_policy = sub_policies[--j];
16589                 mtr_policy = sub_policy->main_policy;
16590                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16591                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16592                         sub_policy_num = (mtr_policy->sub_policy_num >>
16593                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16594                                 MLX5_MTR_SUB_POLICY_NUM_MASK;
16595                         mtr_policy->sub_policys[domain][sub_policy_num - 1] =
16596                                                                         NULL;
16597                         sub_policy_num--;
16598                         mtr_policy->sub_policy_num &=
16599                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16600                                   (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
16601                         mtr_policy->sub_policy_num |=
16602                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16603                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
16604                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16605                                         sub_policy->idx);
16606                 }
16607         }
16608         return NULL;
16609 }
16610
16611 /**
16612  * Create the sub policy tag rule for all meters in hierarchy.
16613  *
16614  * @param[in] dev
16615  *   Pointer to Ethernet device.
16616  * @param[in] fm
16617  *   Meter information table.
16618  * @param[in] src_port
16619  *   The src port this extra rule should use.
16620  * @param[in] item
16621  *   The src port match item.
16622  * @param[out] error
16623  *   Perform verbose error reporting if not NULL.
16624  * @return
16625  *   0 on success, a negative errno value otherwise and rte_errno is set.
16626  */
16627 static int
16628 flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
16629                                 struct mlx5_flow_meter_info *fm,
16630                                 int32_t src_port,
16631                                 const struct rte_flow_item *item,
16632                                 struct rte_flow_error *error)
16633 {
16634         struct mlx5_priv *priv = dev->data->dev_private;
16635         struct mlx5_flow_meter_policy *mtr_policy;
16636         struct mlx5_flow_meter_sub_policy *sub_policy;
16637         struct mlx5_flow_meter_info *next_fm = NULL;
16638         struct mlx5_flow_meter_policy *next_policy;
16639         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16640         struct mlx5_flow_tbl_data_entry *tbl_data;
16641         struct mlx5_sub_policy_color_rule *color_rule;
16642         struct mlx5_meter_policy_acts acts;
16643         uint32_t color_reg_c_idx;
16644         bool mtr_first = (src_port != UINT16_MAX) ? true : false;
16645         struct rte_flow_attr attr = {
16646                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16647                 .priority = 0,
16648                 .ingress = 0,
16649                 .egress = 0,
16650                 .transfer = 1,
16651                 .reserved = 0,
16652         };
16653         uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
16654         int i;
16655
16656         mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
16657         MLX5_ASSERT(mtr_policy);
16658         if (!mtr_policy->is_hierarchy)
16659                 return 0;
16660         next_fm = mlx5_flow_meter_find(priv,
16661                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16662         if (!next_fm) {
16663                 return rte_flow_error_set(error, EINVAL,
16664                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
16665                                 "Failed to find next meter in hierarchy.");
16666         }
16667         if (!next_fm->drop_cnt)
16668                 goto exit;
16669         color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error);
16670         sub_policy = mtr_policy->sub_policys[domain][0];
16671         for (i = 0; i < RTE_COLORS; i++) {
16672                 bool rule_exist = false;
16673                 struct mlx5_meter_policy_action_container *act_cnt;
16674
16675                 if (i >= RTE_COLOR_YELLOW)
16676                         break;
16677                 TAILQ_FOREACH(color_rule,
16678                               &sub_policy->color_rules[i], next_port)
16679                         if (color_rule->src_port == src_port) {
16680                                 rule_exist = true;
16681                                 break;
16682                         }
16683                 if (rule_exist)
16684                         continue;
16685                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
16686                                 sizeof(struct mlx5_sub_policy_color_rule),
16687                                 0, SOCKET_ID_ANY);
16688                 if (!color_rule)
16689                         return rte_flow_error_set(error, ENOMEM,
16690                                 RTE_FLOW_ERROR_TYPE_ACTION,
16691                                 NULL, "No memory to create tag color rule.");
16692                 color_rule->src_port = src_port;
16693                 attr.priority = i;
16694                 next_policy = mlx5_flow_meter_policy_find(dev,
16695                                                 next_fm->policy_id, NULL);
16696                 MLX5_ASSERT(next_policy);
16697                 next_sub_policy = next_policy->sub_policys[domain][0];
16698                 tbl_data = container_of(next_sub_policy->tbl_rsc,
16699                                         struct mlx5_flow_tbl_data_entry, tbl);
16700                 act_cnt = &mtr_policy->act_cnt[i];
16701                 if (mtr_first) {
16702                         acts.dv_actions[0] = next_fm->meter_action;
16703                         acts.dv_actions[1] = act_cnt->modify_hdr->action;
16704                 } else {
16705                         acts.dv_actions[0] = act_cnt->modify_hdr->action;
16706                         acts.dv_actions[1] = next_fm->meter_action;
16707                 }
16708                 acts.dv_actions[2] = tbl_data->jump.action;
16709                 acts.actions_n = 3;
16710                 if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) {
16711                         next_fm = NULL;
16712                         goto err_exit;
16713                 }
16714                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
16715                                         i, sub_policy, &attr, true, item,
16716                                         &color_rule->matcher, error)) {
16717                         rte_flow_error_set(error, errno,
16718                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
16719                                 "Failed to create hierarchy meter matcher.");
16720                         goto err_exit;
16721                 }
16722                 if (__flow_dv_create_policy_flow(dev, color_reg_c_idx,
16723                                         (enum rte_color)i,
16724                                         color_rule->matcher->matcher_object,
16725                                         acts.actions_n, acts.dv_actions,
16726                                         true, item,
16727                                         &color_rule->rule, &attr)) {
16728                         rte_flow_error_set(error, errno,
16729                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
16730                                 "Failed to create hierarchy meter rule.");
16731                         goto err_exit;
16732                 }
16733                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
16734                                   color_rule, next_port);
16735         }
16736 exit:
16737         /**
16738          * Recursive call to iterate all meters in hierarchy and
16739          * create needed rules.
16740          */
16741         return flow_dv_meter_hierarchy_rule_create(dev, next_fm,
16742                                                 src_port, item, error);
16743 err_exit:
16744         if (color_rule) {
16745                 if (color_rule->rule)
16746                         mlx5_flow_os_destroy_flow(color_rule->rule);
16747                 if (color_rule->matcher) {
16748                         struct mlx5_flow_tbl_data_entry *tbl =
16749                                 container_of(color_rule->matcher->tbl,
16750                                                 typeof(*tbl), tbl);
16751                         mlx5_list_unregister(tbl->matchers,
16752                                                 &color_rule->matcher->entry);
16753                 }
16754                 mlx5_free(color_rule);
16755         }
16756         if (next_fm)
16757                 mlx5_flow_meter_detach(priv, next_fm);
16758         return -rte_errno;
16759 }
16760
16761 /**
16762  * Destroy the sub policy table with RX queue.
16763  *
16764  * @param[in] dev
16765  *   Pointer to Ethernet device.
16766  * @param[in] mtr_policy
16767  *   Pointer to meter policy table.
16768  */
16769 static void
16770 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
16771                 struct mlx5_flow_meter_policy *mtr_policy)
16772 {
16773         struct mlx5_priv *priv = dev->data->dev_private;
16774         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16775         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16776         uint32_t i, j;
16777         uint16_t sub_policy_num, new_policy_num;
16778
16779         rte_spinlock_lock(&mtr_policy->sl);
16780         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16781                 switch (mtr_policy->act_cnt[i].fate_action) {
16782                 case MLX5_FLOW_FATE_SHARED_RSS:
16783                         sub_policy_num = (mtr_policy->sub_policy_num >>
16784                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16785                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16786                         new_policy_num = sub_policy_num;
16787                         for (j = 0; j < sub_policy_num; j++) {
16788                                 sub_policy =
16789                                         mtr_policy->sub_policys[domain][j];
16790                                 if (sub_policy) {
16791                                         __flow_dv_destroy_sub_policy_rules(dev,
16792                                                 sub_policy);
16793                                 if (sub_policy !=
16794                                         mtr_policy->sub_policys[domain][0]) {
16795                                         mtr_policy->sub_policys[domain][j] =
16796                                                                 NULL;
16797                                         mlx5_ipool_free
16798                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16799                                                 sub_policy->idx);
16800                                                 new_policy_num--;
16801                                         }
16802                                 }
16803                         }
16804                         if (new_policy_num != sub_policy_num) {
16805                                 mtr_policy->sub_policy_num &=
16806                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16807                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16808                                 mtr_policy->sub_policy_num |=
16809                                 (new_policy_num &
16810                                         MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16811                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16812                         }
16813                         break;
16814                 case MLX5_FLOW_FATE_QUEUE:
16815                         sub_policy = mtr_policy->sub_policys[domain][0];
16816                         __flow_dv_destroy_sub_policy_rules(dev,
16817                                                 sub_policy);
16818                         break;
16819                 default:
16820                         /*Other actions without queue and do nothing*/
16821                         break;
16822                 }
16823         }
16824         rte_spinlock_unlock(&mtr_policy->sl);
16825 }
16826
16827 /**
16828  * Validate the batch counter support in root table.
16829  *
16830  * Create a simple flow with invalid counter and drop action on root table to
16831  * validate if batch counter with offset on root table is supported or not.
16832  *
16833  * @param[in] dev
16834  *   Pointer to rte_eth_dev structure.
16835  *
16836  * @return
16837  *   0 on success, a negative errno value otherwise and rte_errno is set.
16838  */
16839 int
16840 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
16841 {
16842         struct mlx5_priv *priv = dev->data->dev_private;
16843         struct mlx5_dev_ctx_shared *sh = priv->sh;
16844         struct mlx5_flow_dv_match_params mask = {
16845                 .size = sizeof(mask.buf),
16846         };
16847         struct mlx5_flow_dv_match_params value = {
16848                 .size = sizeof(value.buf),
16849         };
16850         struct mlx5dv_flow_matcher_attr dv_attr = {
16851                 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS,
16852                 .priority = 0,
16853                 .match_criteria_enable = 0,
16854                 .match_mask = (void *)&mask,
16855         };
16856         void *actions[2] = { 0 };
16857         struct mlx5_flow_tbl_resource *tbl = NULL;
16858         struct mlx5_devx_obj *dcs = NULL;
16859         void *matcher = NULL;
16860         void *flow = NULL;
16861         int ret = -1;
16862
16863         tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL,
16864                                         0, 0, 0, NULL);
16865         if (!tbl)
16866                 goto err;
16867         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
16868         if (!dcs)
16869                 goto err;
16870         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
16871                                                     &actions[0]);
16872         if (ret)
16873                 goto err;
16874         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
16875         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
16876         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
16877                                                &matcher);
16878         if (ret)
16879                 goto err;
16880         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
16881         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
16882                                        actions, &flow);
16883 err:
16884         /*
16885          * If batch counter with offset is not supported, the driver will not
16886          * validate the invalid offset value, flow create should success.
16887          * In this case, it means batch counter is not supported in root table.
16888          *
16889          * Otherwise, if flow create is failed, counter offset is supported.
16890          */
16891         if (flow) {
16892                 DRV_LOG(INFO, "Batch counter is not supported in root "
16893                               "table. Switch to fallback mode.");
16894                 rte_errno = ENOTSUP;
16895                 ret = -rte_errno;
16896                 claim_zero(mlx5_flow_os_destroy_flow(flow));
16897         } else {
16898                 /* Check matcher to make sure validate fail at flow create. */
16899                 if (!matcher || (matcher && errno != EINVAL))
16900                         DRV_LOG(ERR, "Unexpected error in counter offset "
16901                                      "support detection");
16902                 ret = 0;
16903         }
16904         if (actions[0])
16905                 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0]));
16906         if (matcher)
16907                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
16908         if (tbl)
16909                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
16910         if (dcs)
16911                 claim_zero(mlx5_devx_cmd_destroy(dcs));
16912         return ret;
16913 }
16914
16915 /**
16916  * Query a devx counter.
16917  *
16918  * @param[in] dev
16919  *   Pointer to the Ethernet device structure.
16920  * @param[in] cnt
16921  *   Index to the flow counter.
16922  * @param[in] clear
16923  *   Set to clear the counter statistics.
16924  * @param[out] pkts
16925  *   The statistics value of packets.
16926  * @param[out] bytes
16927  *   The statistics value of bytes.
16928  *
16929  * @return
16930  *   0 on success, otherwise return -1.
16931  */
16932 static int
16933 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
16934                       uint64_t *pkts, uint64_t *bytes)
16935 {
16936         struct mlx5_priv *priv = dev->data->dev_private;
16937         struct mlx5_flow_counter *cnt;
16938         uint64_t inn_pkts, inn_bytes;
16939         int ret;
16940
16941         if (!priv->config.devx)
16942                 return -1;
16943
16944         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
16945         if (ret)
16946                 return -1;
16947         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
16948         *pkts = inn_pkts - cnt->hits;
16949         *bytes = inn_bytes - cnt->bytes;
16950         if (clear) {
16951                 cnt->hits = inn_pkts;
16952                 cnt->bytes = inn_bytes;
16953         }
16954         return 0;
16955 }
16956
16957 /**
16958  * Get aged-out flows.
16959  *
16960  * @param[in] dev
16961  *   Pointer to the Ethernet device structure.
16962  * @param[in] context
16963  *   The address of an array of pointers to the aged-out flows contexts.
16964  * @param[in] nb_contexts
16965  *   The length of context array pointers.
16966  * @param[out] error
16967  *   Perform verbose error reporting if not NULL. Initialized in case of
16968  *   error only.
16969  *
16970  * @return
16971  *   how many contexts get in success, otherwise negative errno value.
16972  *   if nb_contexts is 0, return the amount of all aged contexts.
16973  *   if nb_contexts is not 0 , return the amount of aged flows reported
16974  *   in the context array.
16975  * @note: only stub for now
16976  */
16977 static int
16978 flow_get_aged_flows(struct rte_eth_dev *dev,
16979                     void **context,
16980                     uint32_t nb_contexts,
16981                     struct rte_flow_error *error)
16982 {
16983         struct mlx5_priv *priv = dev->data->dev_private;
16984         struct mlx5_age_info *age_info;
16985         struct mlx5_age_param *age_param;
16986         struct mlx5_flow_counter *counter;
16987         struct mlx5_aso_age_action *act;
16988         int nb_flows = 0;
16989
16990         if (nb_contexts && !context)
16991                 return rte_flow_error_set(error, EINVAL,
16992                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16993                                           NULL, "empty context");
16994         age_info = GET_PORT_AGE_INFO(priv);
16995         rte_spinlock_lock(&age_info->aged_sl);
16996         LIST_FOREACH(act, &age_info->aged_aso, next) {
16997                 nb_flows++;
16998                 if (nb_contexts) {
16999                         context[nb_flows - 1] =
17000                                                 act->age_params.context;
17001                         if (!(--nb_contexts))
17002                                 break;
17003                 }
17004         }
17005         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
17006                 nb_flows++;
17007                 if (nb_contexts) {
17008                         age_param = MLX5_CNT_TO_AGE(counter);
17009                         context[nb_flows - 1] = age_param->context;
17010                         if (!(--nb_contexts))
17011                                 break;
17012                 }
17013         }
17014         rte_spinlock_unlock(&age_info->aged_sl);
17015         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
17016         return nb_flows;
17017 }
17018
17019 /*
17020  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
17021  */
17022 static uint32_t
17023 flow_dv_counter_allocate(struct rte_eth_dev *dev)
17024 {
17025         return flow_dv_counter_alloc(dev, 0);
17026 }
17027
17028 /**
17029  * Validate indirect action.
17030  * Dispatcher for action type specific validation.
17031  *
17032  * @param[in] dev
17033  *   Pointer to the Ethernet device structure.
17034  * @param[in] conf
17035  *   Indirect action configuration.
17036  * @param[in] action
17037  *   The indirect action object to validate.
17038  * @param[out] error
17039  *   Perform verbose error reporting if not NULL. Initialized in case of
17040  *   error only.
17041  *
17042  * @return
17043  *   0 on success, otherwise negative errno value.
17044  */
17045 static int
17046 flow_dv_action_validate(struct rte_eth_dev *dev,
17047                         const struct rte_flow_indir_action_conf *conf,
17048                         const struct rte_flow_action *action,
17049                         struct rte_flow_error *err)
17050 {
17051         struct mlx5_priv *priv = dev->data->dev_private;
17052
17053         RTE_SET_USED(conf);
17054         switch (action->type) {
17055         case RTE_FLOW_ACTION_TYPE_RSS:
17056                 /*
17057                  * priv->obj_ops is set according to driver capabilities.
17058                  * When DevX capabilities are
17059                  * sufficient, it is set to devx_obj_ops.
17060                  * Otherwise, it is set to ibv_obj_ops.
17061                  * ibv_obj_ops doesn't support ind_table_modify operation.
17062                  * In this case the indirect RSS action can't be used.
17063                  */
17064                 if (priv->obj_ops.ind_table_modify == NULL)
17065                         return rte_flow_error_set
17066                                         (err, ENOTSUP,
17067                                          RTE_FLOW_ERROR_TYPE_ACTION,
17068                                          NULL,
17069                                          "Indirect RSS action not supported");
17070                 return mlx5_validate_action_rss(dev, action, err);
17071         case RTE_FLOW_ACTION_TYPE_AGE:
17072                 if (!priv->sh->aso_age_mng)
17073                         return rte_flow_error_set(err, ENOTSUP,
17074                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17075                                                 NULL,
17076                                                 "Indirect age action not supported");
17077                 return flow_dv_validate_action_age(0, action, dev, err);
17078         case RTE_FLOW_ACTION_TYPE_COUNT:
17079                 /*
17080                  * There are two mechanisms to share the action count.
17081                  * The old mechanism uses the shared field to share, while the
17082                  * new mechanism uses the indirect action API.
17083                  * This validation comes to make sure that the two mechanisms
17084                  * are not combined.
17085                  */
17086                 if (is_shared_action_count(action))
17087                         return rte_flow_error_set(err, ENOTSUP,
17088                                                   RTE_FLOW_ERROR_TYPE_ACTION,
17089                                                   NULL,
17090                                                   "Mix shared and indirect counter is not supported");
17091                 return flow_dv_validate_action_count(dev, true, 0, err);
17092         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
17093                 if (!priv->sh->ct_aso_en)
17094                         return rte_flow_error_set(err, ENOTSUP,
17095                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17096                                         "ASO CT is not supported");
17097                 return mlx5_validate_action_ct(dev, action->conf, err);
17098         default:
17099                 return rte_flow_error_set(err, ENOTSUP,
17100                                           RTE_FLOW_ERROR_TYPE_ACTION,
17101                                           NULL,
17102                                           "action type not supported");
17103         }
17104 }
17105
17106 /**
17107  * Validate the meter hierarchy chain for meter policy.
17108  *
17109  * @param[in] dev
17110  *   Pointer to the Ethernet device structure.
17111  * @param[in] meter_id
17112  *   Meter id.
17113  * @param[in] action_flags
17114  *   Holds the actions detected until now.
17115  * @param[out] is_rss
17116  *   Is RSS or not.
17117  * @param[out] hierarchy_domain
17118  *   The domain bitmap for hierarchy policy.
17119  * @param[out] error
17120  *   Perform verbose error reporting if not NULL. Initialized in case of
17121  *   error only.
17122  *
17123  * @return
17124  *   0 on success, otherwise negative errno value with error set.
17125  */
17126 static int
17127 flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
17128                                   uint32_t meter_id,
17129                                   uint64_t action_flags,
17130                                   bool *is_rss,
17131                                   uint8_t *hierarchy_domain,
17132                                   struct rte_mtr_error *error)
17133 {
17134         struct mlx5_priv *priv = dev->data->dev_private;
17135         struct mlx5_flow_meter_info *fm;
17136         struct mlx5_flow_meter_policy *policy;
17137         uint8_t cnt = 1;
17138
17139         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
17140                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17141                 return -rte_mtr_error_set(error, EINVAL,
17142                                         RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN,
17143                                         NULL,
17144                                         "Multiple fate actions not supported.");
17145         while (true) {
17146                 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
17147                 if (!fm)
17148                         return -rte_mtr_error_set(error, EINVAL,
17149                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17150                                         "Meter not found in meter hierarchy.");
17151                 if (fm->def_policy)
17152                         return -rte_mtr_error_set(error, EINVAL,
17153                                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17154                         "Non termination meter not supported in hierarchy.");
17155                 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
17156                 MLX5_ASSERT(policy);
17157                 if (!policy->is_hierarchy) {
17158                         if (policy->transfer)
17159                                 *hierarchy_domain |=
17160                                                 MLX5_MTR_DOMAIN_TRANSFER_BIT;
17161                         if (policy->ingress)
17162                                 *hierarchy_domain |=
17163                                                 MLX5_MTR_DOMAIN_INGRESS_BIT;
17164                         if (policy->egress)
17165                                 *hierarchy_domain |= MLX5_MTR_DOMAIN_EGRESS_BIT;
17166                         *is_rss = policy->is_rss;
17167                         break;
17168                 }
17169                 meter_id = policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id;
17170                 if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM)
17171                         return -rte_mtr_error_set(error, EINVAL,
17172                                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
17173                                         "Exceed max hierarchy meter number.");
17174         }
17175         return 0;
17176 }
17177
17178 /**
17179  * Validate meter policy actions.
17180  * Dispatcher for action type specific validation.
17181  *
17182  * @param[in] dev
17183  *   Pointer to the Ethernet device structure.
17184  * @param[in] action
17185  *   The meter policy action object to validate.
17186  * @param[in] attr
17187  *   Attributes of flow to determine steering domain.
17188  * @param[out] error
17189  *   Perform verbose error reporting if not NULL. Initialized in case of
17190  *   error only.
17191  *
17192  * @return
17193  *   0 on success, otherwise negative errno value.
17194  */
17195 static int
17196 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
17197                         const struct rte_flow_action *actions[RTE_COLORS],
17198                         struct rte_flow_attr *attr,
17199                         bool *is_rss,
17200                         uint8_t *domain_bitmap,
17201                         bool *is_def_policy,
17202                         struct rte_mtr_error *error)
17203 {
17204         struct mlx5_priv *priv = dev->data->dev_private;
17205         struct mlx5_dev_config *dev_conf = &priv->config;
17206         const struct rte_flow_action *act;
17207         uint64_t action_flags = 0;
17208         int actions_n;
17209         int i, ret;
17210         struct rte_flow_error flow_err;
17211         uint8_t domain_color[RTE_COLORS] = {0};
17212         uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
17213         uint8_t hierarchy_domain = 0;
17214         const struct rte_flow_action_meter *mtr;
17215
17216         if (!priv->config.dv_esw_en)
17217                 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
17218         *domain_bitmap = def_domain;
17219         if (actions[RTE_COLOR_YELLOW] &&
17220                 actions[RTE_COLOR_YELLOW]->type != RTE_FLOW_ACTION_TYPE_END)
17221                 return -rte_mtr_error_set(error, ENOTSUP,
17222                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17223                                 NULL,
17224                                 "Yellow color does not support any action.");
17225         if (actions[RTE_COLOR_YELLOW] &&
17226                 actions[RTE_COLOR_YELLOW]->type != RTE_FLOW_ACTION_TYPE_DROP)
17227                 return -rte_mtr_error_set(error, ENOTSUP,
17228                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17229                                 NULL, "Red color only supports drop action.");
17230         /*
17231          * Check default policy actions:
17232          * Green/Yellow: no action, Red: drop action
17233          */
17234         if ((!actions[RTE_COLOR_GREEN] ||
17235                 actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END)) {
17236                 *is_def_policy = true;
17237                 return 0;
17238         }
17239         flow_err.message = NULL;
17240         for (i = 0; i < RTE_COLORS; i++) {
17241                 act = actions[i];
17242                 for (action_flags = 0, actions_n = 0;
17243                         act && act->type != RTE_FLOW_ACTION_TYPE_END;
17244                         act++) {
17245                         if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
17246                                 return -rte_mtr_error_set(error, ENOTSUP,
17247                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17248                                           NULL, "too many actions");
17249                         switch (act->type) {
17250                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
17251                                 if (!priv->config.dv_esw_en)
17252                                         return -rte_mtr_error_set(error,
17253                                         ENOTSUP,
17254                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17255                                         NULL, "PORT action validate check"
17256                                         " fail for ESW disable");
17257                                 ret = flow_dv_validate_action_port_id(dev,
17258                                                 action_flags,
17259                                                 act, attr, &flow_err);
17260                                 if (ret)
17261                                         return -rte_mtr_error_set(error,
17262                                         ENOTSUP,
17263                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17264                                         NULL, flow_err.message ?
17265                                         flow_err.message :
17266                                         "PORT action validate check fail");
17267                                 ++actions_n;
17268                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
17269                                 break;
17270                         case RTE_FLOW_ACTION_TYPE_MARK:
17271                                 ret = flow_dv_validate_action_mark(dev, act,
17272                                                            action_flags,
17273                                                            attr, &flow_err);
17274                                 if (ret < 0)
17275                                         return -rte_mtr_error_set(error,
17276                                         ENOTSUP,
17277                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17278                                         NULL, flow_err.message ?
17279                                         flow_err.message :
17280                                         "Mark action validate check fail");
17281                                 if (dev_conf->dv_xmeta_en !=
17282                                         MLX5_XMETA_MODE_LEGACY)
17283                                         return -rte_mtr_error_set(error,
17284                                         ENOTSUP,
17285                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17286                                         NULL, "Extend MARK action is "
17287                                         "not supported. Please try use "
17288                                         "default policy for meter.");
17289                                 action_flags |= MLX5_FLOW_ACTION_MARK;
17290                                 ++actions_n;
17291                                 break;
17292                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
17293                                 ret = flow_dv_validate_action_set_tag(dev,
17294                                                         act, action_flags,
17295                                                         attr, &flow_err);
17296                                 if (ret)
17297                                         return -rte_mtr_error_set(error,
17298                                         ENOTSUP,
17299                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17300                                         NULL, flow_err.message ?
17301                                         flow_err.message :
17302                                         "Set tag action validate check fail");
17303                                 /*
17304                                  * Count all modify-header actions
17305                                  * as one action.
17306                                  */
17307                                 if (!(action_flags &
17308                                         MLX5_FLOW_MODIFY_HDR_ACTIONS))
17309                                         ++actions_n;
17310                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
17311                                 break;
17312                         case RTE_FLOW_ACTION_TYPE_DROP:
17313                                 ret = mlx5_flow_validate_action_drop
17314                                         (action_flags,
17315                                         attr, &flow_err);
17316                                 if (ret < 0)
17317                                         return -rte_mtr_error_set(error,
17318                                         ENOTSUP,
17319                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17320                                         NULL, flow_err.message ?
17321                                         flow_err.message :
17322                                         "Drop action validate check fail");
17323                                 action_flags |= MLX5_FLOW_ACTION_DROP;
17324                                 ++actions_n;
17325                                 break;
17326                         case RTE_FLOW_ACTION_TYPE_QUEUE:
17327                                 /*
17328                                  * Check whether extensive
17329                                  * metadata feature is engaged.
17330                                  */
17331                                 if (dev_conf->dv_flow_en &&
17332                                         (dev_conf->dv_xmeta_en !=
17333                                         MLX5_XMETA_MODE_LEGACY) &&
17334                                         mlx5_flow_ext_mreg_supported(dev))
17335                                         return -rte_mtr_error_set(error,
17336                                           ENOTSUP,
17337                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17338                                           NULL, "Queue action with meta "
17339                                           "is not supported. Please try use "
17340                                           "default policy for meter.");
17341                                 ret = mlx5_flow_validate_action_queue(act,
17342                                                         action_flags, dev,
17343                                                         attr, &flow_err);
17344                                 if (ret < 0)
17345                                         return -rte_mtr_error_set(error,
17346                                           ENOTSUP,
17347                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17348                                           NULL, flow_err.message ?
17349                                           flow_err.message :
17350                                           "Queue action validate check fail");
17351                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
17352                                 ++actions_n;
17353                                 break;
17354                         case RTE_FLOW_ACTION_TYPE_RSS:
17355                                 if (dev_conf->dv_flow_en &&
17356                                         (dev_conf->dv_xmeta_en !=
17357                                         MLX5_XMETA_MODE_LEGACY) &&
17358                                         mlx5_flow_ext_mreg_supported(dev))
17359                                         return -rte_mtr_error_set(error,
17360                                           ENOTSUP,
17361                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17362                                           NULL, "RSS action with meta "
17363                                           "is not supported. Please try use "
17364                                           "default policy for meter.");
17365                                 ret = mlx5_validate_action_rss(dev, act,
17366                                                 &flow_err);
17367                                 if (ret < 0)
17368                                         return -rte_mtr_error_set(error,
17369                                           ENOTSUP,
17370                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17371                                           NULL, flow_err.message ?
17372                                           flow_err.message :
17373                                           "RSS action validate check fail");
17374                                 action_flags |= MLX5_FLOW_ACTION_RSS;
17375                                 ++actions_n;
17376                                 *is_rss = true;
17377                                 break;
17378                         case RTE_FLOW_ACTION_TYPE_JUMP:
17379                                 ret = flow_dv_validate_action_jump(dev,
17380                                         NULL, act, action_flags,
17381                                         attr, true, &flow_err);
17382                                 if (ret)
17383                                         return -rte_mtr_error_set(error,
17384                                           ENOTSUP,
17385                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17386                                           NULL, flow_err.message ?
17387                                           flow_err.message :
17388                                           "Jump action validate check fail");
17389                                 ++actions_n;
17390                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
17391                                 break;
17392                         case RTE_FLOW_ACTION_TYPE_METER:
17393                                 if (i != RTE_COLOR_GREEN)
17394                                         return -rte_mtr_error_set(error,
17395                                                 ENOTSUP,
17396                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17397                                                 NULL, flow_err.message ?
17398                                                 flow_err.message :
17399                                   "Meter hierarchy only supports GREEN color.");
17400                                 mtr = act->conf;
17401                                 ret = flow_dv_validate_policy_mtr_hierarchy(dev,
17402                                                         mtr->mtr_id,
17403                                                         action_flags,
17404                                                         is_rss,
17405                                                         &hierarchy_domain,
17406                                                         error);
17407                                 if (ret)
17408                                         return ret;
17409                                 ++actions_n;
17410                                 action_flags |=
17411                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
17412                                 break;
17413                         default:
17414                                 return -rte_mtr_error_set(error, ENOTSUP,
17415                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17416                                         NULL,
17417                                         "Doesn't support optional action");
17418                         }
17419                 }
17420                 /* Yellow is not supported, just skip. */
17421                 if (i == RTE_COLOR_YELLOW)
17422                         continue;
17423                 if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
17424                         domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT;
17425                 else if ((action_flags &
17426                         (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) ||
17427                         (action_flags & MLX5_FLOW_ACTION_MARK))
17428                         /*
17429                          * Only support MLX5_XMETA_MODE_LEGACY
17430                          * so MARK action only in ingress domain.
17431                          */
17432                         domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT;
17433                 else if (action_flags &
17434                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
17435                         domain_color[i] = hierarchy_domain;
17436                 else
17437                         domain_color[i] = def_domain;
17438                 /*
17439                  * Validate the drop action mutual exclusion
17440                  * with other actions. Drop action is mutually-exclusive
17441                  * with any other action, except for Count action.
17442                  */
17443                 if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
17444                         (action_flags & ~MLX5_FLOW_ACTION_DROP)) {
17445                         return -rte_mtr_error_set(error, ENOTSUP,
17446                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17447                                 NULL, "Drop action is mutually-exclusive "
17448                                 "with any other action");
17449                 }
17450                 /* Eswitch has few restrictions on using items and actions */
17451                 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) {
17452                         if (!mlx5_flow_ext_mreg_supported(dev) &&
17453                                 action_flags & MLX5_FLOW_ACTION_MARK)
17454                                 return -rte_mtr_error_set(error, ENOTSUP,
17455                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17456                                         NULL, "unsupported action MARK");
17457                         if (action_flags & MLX5_FLOW_ACTION_QUEUE)
17458                                 return -rte_mtr_error_set(error, ENOTSUP,
17459                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17460                                         NULL, "unsupported action QUEUE");
17461                         if (action_flags & MLX5_FLOW_ACTION_RSS)
17462                                 return -rte_mtr_error_set(error, ENOTSUP,
17463                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17464                                         NULL, "unsupported action RSS");
17465                         if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17466                                 return -rte_mtr_error_set(error, ENOTSUP,
17467                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17468                                         NULL, "no fate action is found");
17469                 } else {
17470                         if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) &&
17471                                 (domain_color[i] &
17472                                 MLX5_MTR_DOMAIN_INGRESS_BIT)) {
17473                                 if ((domain_color[i] &
17474                                         MLX5_MTR_DOMAIN_EGRESS_BIT))
17475                                         domain_color[i] =
17476                                         MLX5_MTR_DOMAIN_EGRESS_BIT;
17477                                 else
17478                                         return -rte_mtr_error_set(error,
17479                                         ENOTSUP,
17480                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17481                                         NULL, "no fate action is found");
17482                         }
17483                 }
17484                 if (domain_color[i] != def_domain)
17485                         *domain_bitmap = domain_color[i];
17486         }
17487         return 0;
17488 }
17489
17490 static int
17491 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
17492 {
17493         struct mlx5_priv *priv = dev->data->dev_private;
17494         int ret = 0;
17495
17496         if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
17497                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain,
17498                                                 flags);
17499                 if (ret != 0)
17500                         return ret;
17501         }
17502         if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
17503                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags);
17504                 if (ret != 0)
17505                         return ret;
17506         }
17507         if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
17508                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags);
17509                 if (ret != 0)
17510                         return ret;
17511         }
17512         return 0;
17513 }
17514
17515 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
17516         .validate = flow_dv_validate,
17517         .prepare = flow_dv_prepare,
17518         .translate = flow_dv_translate,
17519         .apply = flow_dv_apply,
17520         .remove = flow_dv_remove,
17521         .destroy = flow_dv_destroy,
17522         .query = flow_dv_query,
17523         .create_mtr_tbls = flow_dv_create_mtr_tbls,
17524         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls,
17525         .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls,
17526         .create_meter = flow_dv_mtr_alloc,
17527         .free_meter = flow_dv_aso_mtr_release_to_pool,
17528         .validate_mtr_acts = flow_dv_validate_mtr_policy_acts,
17529         .create_mtr_acts = flow_dv_create_mtr_policy_acts,
17530         .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts,
17531         .create_policy_rules = flow_dv_create_policy_rules,
17532         .destroy_policy_rules = flow_dv_destroy_policy_rules,
17533         .create_def_policy = flow_dv_create_def_policy,
17534         .destroy_def_policy = flow_dv_destroy_def_policy,
17535         .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare,
17536         .meter_hierarchy_rule_create = flow_dv_meter_hierarchy_rule_create,
17537         .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq,
17538         .counter_alloc = flow_dv_counter_allocate,
17539         .counter_free = flow_dv_counter_free,
17540         .counter_query = flow_dv_counter_query,
17541         .get_aged_flows = flow_get_aged_flows,
17542         .action_validate = flow_dv_action_validate,
17543         .action_create = flow_dv_action_create,
17544         .action_destroy = flow_dv_action_destroy,
17545         .action_update = flow_dv_action_update,
17546         .action_query = flow_dv_action_query,
17547         .sync_domain = flow_dv_sync_domain,
17548 };
17549
17550 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
17551